aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorEugene Triguba <hello@eugenetriguba.com>2025-10-11 14:19:39 -0400
committerEugene Triguba <hello@eugenetriguba.com>2025-10-11 14:31:37 -0400
commitd3ec525b7d6ecd8de064d3f3a54d112769d1d3b3 (patch)
tree935b51f95afb513604c82c9e1efca223af60c109
parentFix git clone link in readme (diff)
downloadcli-d3ec525b7d6ecd8de064d3f3a54d112769d1d3b3.zip
Add unicode support to justify
-rw-r--r--CHANGELOG.md6
-rw-r--r--CMakeLists.txt6
-rw-r--r--Dockerfile4
-rw-r--r--doc/justify.14
-rw-r--r--include/linebuffer.h8
-rw-r--r--include/utf8.h14
-rw-r--r--src/linebuffer.c27
-rw-r--r--src/utf8.c74
-rw-r--r--tests/CMakeLists.txt9
-rw-r--r--tests/justify.bats39
-rw-r--r--tests/test_linebuffer.c10
-rw-r--r--tests/test_utf8.c118
12 files changed, 298 insertions, 21 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1f67106..845932f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [Unreleased]
+
+### Added
+
+- UTF-8 Unicode support for proper text justification with multi-byte characters
+
## 0.1.3 - 2025-06-12
### Changed
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b03d8e2..ef829da 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -11,14 +11,20 @@ if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
add_compile_options(-Wall -Wextra -ansi)
endif()
+# Find ICU library using pkg-config
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(ICU REQUIRED IMPORTED_TARGET icu-uc icu-i18n)
+
set(SRC_FILES
src/linebuffer.c
src/word.c
+ src/utf8.c
src/main.c
)
include_directories(src)
add_executable(justify ${SRC_FILES})
+target_link_libraries(justify PkgConfig::ICU)
install(TARGETS justify RUNTIME DESTINATION bin)
install(FILES doc/justify.1 DESTINATION share/man/man1)
diff --git a/Dockerfile b/Dockerfile
index acc515c..7a37d93 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -12,7 +12,9 @@ RUN apk update && \
cmake \
valgrind \
bash \
- vim
+ vim \
+ icu-dev \
+ icu-libs
WORKDIR /app
COPY . /app
diff --git a/doc/justify.1 b/doc/justify.1
index 1b7a4f6..cba6024 100644
--- a/doc/justify.1
+++ b/doc/justify.1
@@ -44,8 +44,8 @@ An error occurred (e.g., read error).
.SH NOTES
Before appending a word to the current line, the program ensures there is enough space for the word (and a space if not the first word). If not enough space remains, the current line is flushed (justified) and a new line is started.
-.SH LIMITATIONS
-The program assumes ASCII text. This means if multi-byte characters are provided, the output would not be justified correctly.
+.SH UNICODE SUPPORT
+The program supports UTF-8 encoded text and correctly handles multi-byte characters when calculating line widths. Display width is determined using Unicode East Asian Width properties, so full-width and wide characters (such as CJK characters) are counted as taking two columns, while most other characters take one column.
.SH AUTHOR
Eugene Triguba <hello@eugenetriguba.com>
diff --git a/include/linebuffer.h b/include/linebuffer.h
index a3d9e4c..b11cb8f 100644
--- a/include/linebuffer.h
+++ b/include/linebuffer.h
@@ -6,9 +6,11 @@
#include <stdio.h>
typedef struct {
- size_t _num_words;
- size_t _buffer_length;
- size_t _buffer_capacity;
+ size_t _num_words; // Number of words in the buffer
+ size_t _line_width; // Max display width (e.g., 72 columns)
+ size_t _buffer_length; // Current bytes used
+ size_t _display_width; // Current display width
+ size_t _buffer_capacity; // Byte capacity (line_width * 4 for UTF-8)
char *_buffer;
} LineBuffer;
diff --git a/include/utf8.h b/include/utf8.h
new file mode 100644
index 0000000..aa2cac5
--- /dev/null
+++ b/include/utf8.h
@@ -0,0 +1,14 @@
+#ifndef UTF8_H
+#define UTF8_H
+
+#include <stddef.h>
+
+/**
+ * @brief Calculate the display width of a UTF-8 string
+ *
+ * @param str The UTF-8 encoded string
+ * @return The number of characters (display width)
+ */
+size_t utf8_display_width(const char *str);
+
+#endif
diff --git a/src/linebuffer.c b/src/linebuffer.c
index 6773ca1..3fa08b0 100644
--- a/src/linebuffer.c
+++ b/src/linebuffer.c
@@ -1,4 +1,5 @@
#include "linebuffer.h"
+#include "utf8.h"
#include <ctype.h>
#include <stdbool.h>
#include <stdio.h>
@@ -6,19 +7,23 @@
#include <string.h>
#define NULL_TERMINATOR_LEN 1
+#define UTF8_MAX_BYTES_PER_CHAR 4
-LineBuffer *linebuffer_create(size_t buffer_capacity) {
+LineBuffer *linebuffer_create(size_t line_width) {
LineBuffer *lb = malloc(sizeof(LineBuffer));
if (lb == NULL) {
return NULL;
}
+ size_t buffer_capacity = line_width * UTF8_MAX_BYTES_PER_CHAR;
lb->_buffer = malloc(buffer_capacity + NULL_TERMINATOR_LEN);
if (lb->_buffer == NULL) {
free(lb);
return NULL;
}
lb->_num_words = 0;
+ lb->_line_width = line_width;
lb->_buffer_length = 0;
+ lb->_display_width = 0;
lb->_buffer_capacity = buffer_capacity;
return lb;
}
@@ -31,6 +36,7 @@ void linebuffer_destroy(LineBuffer *lb) {
void linebuffer_clear(LineBuffer *lb) {
lb->_num_words = 0;
lb->_buffer_length = 0;
+ lb->_display_width = 0;
}
bool linebuffer_append_word(LineBuffer *lb, const char *word) {
@@ -40,25 +46,28 @@ bool linebuffer_append_word(LineBuffer *lb, const char *word) {
if (lb->_num_words > 0) {
lb->_buffer[lb->_buffer_length] = ' ';
lb->_buffer_length++;
+ lb->_display_width++;
}
- size_t word_length = strlen(word);
- memcpy(lb->_buffer + lb->_buffer_length, word, word_length);
- lb->_buffer_length += word_length;
+ size_t word_byte_length = strlen(word);
+ size_t word_display_width = utf8_display_width(word);
+ memcpy(lb->_buffer + lb->_buffer_length, word, word_byte_length);
+ lb->_buffer_length += word_byte_length;
+ lb->_display_width += word_display_width;
lb->_num_words++;
return true;
}
bool linebuffer_can_fit(LineBuffer *lb, const char *word) {
- size_t append_len = strlen(word);
+ size_t word_display_width = utf8_display_width(word);
+ size_t required_width = word_display_width;
if (lb->_num_words > 0) {
- // Account for space before the word
- append_len++;
+ required_width++;
}
- return append_len <= linebuffer_space_remaining(lb);
+ return required_width <= linebuffer_space_remaining(lb);
}
size_t linebuffer_space_remaining(LineBuffer *lb) {
- return lb->_buffer_capacity - lb->_buffer_length;
+ return lb->_line_width - lb->_display_width;
}
int linebuffer_write(LineBuffer *lb, FILE *fp) {
diff --git a/src/utf8.c b/src/utf8.c
new file mode 100644
index 0000000..07b751e
--- /dev/null
+++ b/src/utf8.c
@@ -0,0 +1,74 @@
+#include "utf8.h"
+#include <unicode/ustring.h>
+#include <unicode/uchar.h>
+#include <unicode/ubrk.h>
+#include <string.h>
+#include <stdlib.h>
+
+size_t utf8_display_width(const char *str) {
+ if (str == NULL) {
+ return 0;
+ }
+
+ UErrorCode status = U_ZERO_ERROR;
+ int32_t utf8_len = (int32_t)strlen(str);
+
+ // Convert UTF-8 to UTF-16 for ICU processing
+ UChar *utf16 = NULL;
+ int32_t utf16_len = 0;
+
+ // First call to get required length
+ u_strFromUTF8(NULL, 0, &utf16_len, str, utf8_len, &status);
+ if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) {
+ return 0;
+ }
+
+ status = U_ZERO_ERROR;
+ utf16 = (UChar *)malloc((utf16_len + 1) * sizeof(UChar));
+ if (utf16 == NULL) {
+ return 0;
+ }
+
+ u_strFromUTF8(utf16, utf16_len + 1, NULL, str, utf8_len, &status);
+ if (U_FAILURE(status)) {
+ free(utf16);
+ return 0;
+ }
+
+ // Use break iterator to count grapheme clusters
+ UBreakIterator *bi = ubrk_open(UBRK_CHARACTER, NULL, utf16, utf16_len, &status);
+ if (U_FAILURE(status)) {
+ free(utf16);
+ return 0;
+ }
+
+ size_t width = 0;
+ int32_t start = ubrk_first(bi);
+ int32_t end = ubrk_next(bi);
+
+ while (end != UBRK_DONE) {
+ // Get the first code point of this grapheme cluster
+ UChar32 ch;
+ int32_t idx = start;
+ U16_NEXT(utf16, idx, utf16_len, ch);
+
+ // Determine display width based on East Asian Width property
+ int32_t ea_width = u_getIntPropertyValue(ch, UCHAR_EAST_ASIAN_WIDTH);
+
+ // Full-width and Wide characters take 2 columns
+ if (ea_width == U_EA_FULLWIDTH || ea_width == U_EA_WIDE) {
+ width += 2;
+ } else {
+ // All others (Narrow, Halfwidth, Neutral, Ambiguous) take 1 column
+ width += 1;
+ }
+
+ start = end;
+ end = ubrk_next(bi);
+ }
+
+ ubrk_close(bi);
+ free(utf16);
+
+ return width;
+}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index a0a8afa..fb75bc6 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -1,14 +1,19 @@
-set(TEST_LINEBUFFER_SOURCES test_linebuffer.c ../src/linebuffer.c)
+set(TEST_LINEBUFFER_SOURCES test_linebuffer.c ../src/linebuffer.c ../src/utf8.c)
set(TEST_WORD_SOURCES test_word.c ../src/word.c)
+set(TEST_UTF8_SOURCES test_utf8.c ../src/utf8.c)
add_executable(test_linebuffer ${TEST_LINEBUFFER_SOURCES})
add_executable(test_word ${TEST_WORD_SOURCES})
+add_executable(test_utf8 ${TEST_UTF8_SOURCES})
add_test(NAME test_linebuffer COMMAND test_linebuffer)
add_test(NAME test_word COMMAND test_word)
+add_test(NAME test_utf8 COMMAND test_utf8)
find_package(PkgConfig REQUIRED)
pkg_check_modules(CMOCKA REQUIRED cmocka)
+pkg_check_modules(ICU REQUIRED IMPORTED_TARGET icu-uc icu-i18n)
include_directories(${CMOCKA_INCLUDE_DIRS})
-target_link_libraries(test_linebuffer ${CMOCKA_LIBRARIES})
+target_link_libraries(test_linebuffer ${CMOCKA_LIBRARIES} PkgConfig::ICU)
target_link_libraries(test_word ${CMOCKA_LIBRARIES})
+target_link_libraries(test_utf8 ${CMOCKA_LIBRARIES} PkgConfig::ICU)
diff --git a/tests/justify.bats b/tests/justify.bats
index 806ea81..525c5d8 100644
--- a/tests/justify.bats
+++ b/tests/justify.bats
@@ -94,3 +94,42 @@ EOF
assert_success
assert_output ""
}
+
+@test "unicode text with accented characters is justified correctly" {
+ run "${JUSTIFY_BIN}" -w 30 <<EOF
+café résumé naïve Zürich another word
+EOF
+ assert_success
+ expected_output="café résumé naïve Zürich
+another word"
+ assert_output "$expected_output"
+}
+
+@test "unicode text with CJK characters is justified correctly" {
+ run "${JUSTIFY_BIN}" -w 20 <<EOF
+Hello 世界 test this is a longer line
+EOF
+ assert_success
+ expected_output="Hello 世界 test this
+is a longer line"
+ assert_output "$expected_output"
+}
+
+@test "mixed ASCII and unicode text wraps correctly" {
+ run "${JUSTIFY_BIN}" -w 25 <<EOF
+This is café and résumé text that should wrap properly
+EOF
+ assert_success
+ expected_output="This is café and résumé
+text that should wrap
+properly"
+ assert_output "$expected_output"
+}
+
+@test "unicode text on last line is left-aligned" {
+ run "${JUSTIFY_BIN}" <<EOF
+café résumé
+EOF
+ assert_success
+ assert_output "café résumé"
+}
diff --git a/tests/test_linebuffer.c b/tests/test_linebuffer.c
index ae233cc..6d84780 100644
--- a/tests/test_linebuffer.c
+++ b/tests/test_linebuffer.c
@@ -10,17 +10,19 @@
#include <string.h>
/**
- * GIVEN a specific buffer capacity
+ * GIVEN a specific line width
* WHEN we attempt to create a LineBuffer
* THEN we expect it to be created successfully
* with the expected internal state
*/
void test_linebuffer_create(void **state) {
- const int buffer_capacity = 50;
- LineBuffer *lb = linebuffer_create(buffer_capacity);
+ const int line_width = 50;
+ LineBuffer *lb = linebuffer_create(line_width);
assert_non_null(lb);
- assert_int_equal(lb->_buffer_capacity, buffer_capacity);
+ assert_int_equal(lb->_line_width, line_width);
+ assert_int_equal(lb->_buffer_capacity, line_width * 4);
assert_int_equal(lb->_buffer_length, 0);
+ assert_int_equal(lb->_display_width, 0);
assert_int_equal(lb->_num_words, 0);
linebuffer_destroy(lb);
}
diff --git a/tests/test_utf8.c b/tests/test_utf8.c
new file mode 100644
index 0000000..623b607
--- /dev/null
+++ b/tests/test_utf8.c
@@ -0,0 +1,118 @@
+#include "utf8.h"
+// clang-format off
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+// clang-format on
+
+/**
+ * GIVEN an ASCII string
+ * WHEN we calculate the display width
+ * THEN we expect the width to equal the byte length
+ */
+void test_utf8_display_width_ascii(void **state) {
+ const char *input = "hello";
+ size_t expected = 5;
+
+ size_t result = utf8_display_width(input);
+
+ assert_int_equal(result, expected);
+}
+
+/**
+ * GIVEN a string with accented characters (combining marks)
+ * WHEN we calculate the display width
+ * THEN we expect the width to be less than the byte length
+ */
+void test_utf8_display_width_accented(void **state) {
+ const char *input = "café"; // é is 2 bytes in UTF-8
+ size_t expected = 4; // 4 visible characters
+
+ size_t result = utf8_display_width(input);
+
+ assert_int_equal(result, expected);
+}
+
+/**
+ * GIVEN a string with East Asian wide characters
+ * WHEN we calculate the display width
+ * THEN we expect the width to be double the character count
+ */
+void test_utf8_display_width_cjk(void **state) {
+ const char *input = "日本語"; // 3 Japanese characters
+ size_t expected = 6; // Each CJK character is 2 columns wide
+
+ size_t result = utf8_display_width(input);
+
+ assert_int_equal(result, expected);
+}
+
+/**
+ * GIVEN a string with mixed ASCII and UTF-8 characters
+ * WHEN we calculate the display width
+ * THEN we expect the correct combined width
+ */
+void test_utf8_display_width_mixed(void **state) {
+ const char *input = "Hello 世界"; // "Hello " (6) + "世界" (4)
+ size_t expected = 10; // 6 ASCII + 2 CJK (2 columns each)
+
+ size_t result = utf8_display_width(input);
+
+ assert_int_equal(result, expected);
+}
+
+/**
+ * GIVEN an empty string
+ * WHEN we calculate the display width
+ * THEN we expect width of 0
+ */
+void test_utf8_display_width_empty(void **state) {
+ const char *input = "";
+ size_t expected = 0;
+
+ size_t result = utf8_display_width(input);
+
+ assert_int_equal(result, expected);
+}
+
+/**
+ * GIVEN a NULL pointer
+ * WHEN we calculate the display width
+ * THEN we expect width of 0
+ */
+void test_utf8_display_width_null(void **state) {
+ size_t expected = 0;
+
+ size_t result = utf8_display_width(NULL);
+
+ assert_int_equal(result, expected);
+}
+
+/**
+ * GIVEN a string with various Unicode characters
+ * WHEN we calculate the display width
+ * THEN we expect the correct width
+ */
+void test_utf8_display_width_various_unicode(void **state) {
+ const char *input = "Zürich"; // ü is 2 bytes in UTF-8
+ size_t expected = 6; // 6 visible characters
+
+ size_t result = utf8_display_width(input);
+
+ assert_int_equal(result, expected);
+}
+
+int main(void) {
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test(test_utf8_display_width_ascii),
+ cmocka_unit_test(test_utf8_display_width_accented),
+ cmocka_unit_test(test_utf8_display_width_cjk),
+ cmocka_unit_test(test_utf8_display_width_mixed),
+ cmocka_unit_test(test_utf8_display_width_empty),
+ cmocka_unit_test(test_utf8_display_width_null),
+ cmocka_unit_test(test_utf8_display_width_various_unicode),
+ };
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}