Download frontend/wasm/src/unicode_engine.rs from Snapkitty/sov-kernel-monster: direct link, hf CLI and curl.
- Browser
- Download file 8.42 kB
-
https://huggingface.co/Snapkitty/sov-kernel-monster/resolve/main/frontend/wasm/src/unicode_engine.rs
- Command line
-
hf download hf://Snapkitty/sov-kernel-monster/frontend/wasm/src/unicode_engine.rs
-
curl -L -o unicode_engine.rs https://huggingface.co/Snapkitty/sov-kernel-monster/resolve/main/frontend/wasm/src/unicode_engine.rs
8.42 kB
| /// Unicode Engine โ deterministic Unicode normalization and analysis | |
| /// Provides NFC/NFKC normalization, code point analysis, bidirectional detection, | |
| /// and roundtrip verification with zero external state. | |
| use wasm_bindgen::prelude::*; | |
| use unicode_normalization::UnicodeNormalization; | |
| use serde_json::{json, Value}; | |
| /// Normalize text to specified Unicode form (NFC or NFKC) | |
| /// | |
| /// # Arguments | |
| /// * `input` - The text to normalize | |
| /// * `form` - "NFC" or "NFKC" (case-insensitive) | |
| /// | |
| /// # Returns | |
| /// Normalized UTF-8 string | |
| pub fn normalize(input: &str, form: &str) -> String { | |
| match form.to_uppercase().as_str() { | |
| "NFC" => input.nfc().collect::<String>(), | |
| "NFKC" => input.nfkc().collect::<String>(), | |
| _ => input.to_string(), // Default: return as-is | |
| } | |
| } | |
| /// Encode text to comprehensive Unicode representation | |
| /// | |
| /// # Returns | |
| /// JSON object with structure: | |
| /// { | |
| /// "normalized": "NFC-normalized text", | |
| /// "codePoints": [array of decimal code points], | |
| /// "utf8Bytes": [array of decimal byte values], | |
| /// "length": number of Unicode scalar values, | |
| /// "byteLength": byte count of UTF-8 encoding | |
| /// } | |
| pub fn encode(input: &str) -> String { | |
| let normalized = input.nfc().collect::<String>(); | |
| let code_points: Vec<u32> = input.chars().map(|c| c as u32).collect(); | |
| let utf8_bytes: Vec<u8> = input.as_bytes().to_vec(); | |
| let length = input.chars().count(); | |
| let byte_length = utf8_bytes.len(); | |
| let result = json!({ | |
| "normalized": normalized, | |
| "codePoints": code_points, | |
| "utf8Bytes": utf8_bytes, | |
| "length": length, | |
| "byteLength": byte_length | |
| }); | |
| result.to_string() | |
| } | |
| /// Detect whether input contains astral plane characters (code points > 0xFFFF) | |
| /// | |
| /// Astral characters require surrogate pairs in UTF-16 and special handling in JS | |
| pub fn has_astral_characters(input: &str) -> bool { | |
| input.chars().any(|c| (c as u32) > 0xFFFF) | |
| } | |
| /// Detect combining marks (Unicode block U+0300โU+036F, also U+1AB0โU+1AFF, etc.) | |
| /// | |
| /// Returns true if input contains any combining diacritical marks | |
| pub fn has_combining_marks(input: &str) -> bool { | |
| input.chars().any(|c| { | |
| let code = c as u32; | |
| // Combining Diacritical Marks: U+0300โU+036F | |
| (code >= 0x0300 && code <= 0x036F) || | |
| // Combining Diacritical Marks Extended: U+1AB0โU+1AFF | |
| (code >= 0x1AB0 && code <= 0x1AFF) || | |
| // Combining Diacritical Marks Supplement: U+1DC0โU+1DFF | |
| (code >= 0x1DC0 && code <= 0x1DFF) || | |
| // Combining Half Marks: U+FE20โU+FE2F | |
| (code >= 0xFE20 && code <= 0xFE2F) | |
| }) | |
| } | |
| /// Detect bidirectional text direction | |
| /// | |
| /// Returns one of: "LTR" (left-to-right), "RTL" (right-to-left), "NEUTRAL" | |
| pub fn detect_bidi_level(input: &str) -> String { | |
| let mut has_rtl = false; | |
| let mut has_ltr = false; | |
| for c in input.chars() { | |
| let code = c as u32; | |
| // RTL ranges: Hebrew, Arabic, Syriac, Thaana, Devanagari (some), etc. | |
| if (code >= 0x0590 && code <= 0x08FF) || // Hebrew + Arabic | |
| (code >= 0xFB1D && code <= 0xFB4F) || // Hebrew presentation forms | |
| (code >= 0xFB50 && code <= 0xFDFF) || // Arabic presentation forms A | |
| (code >= 0xFE70 && code <= 0xFEFF) { // Arabic presentation forms B | |
| has_rtl = true; | |
| } | |
| // LTR ranges: Latin, Greek, Cyrillic, etc. | |
| if (code >= 0x0041 && code <= 0x005A) || // A-Z | |
| (code >= 0x0061 && code <= 0x007A) || // a-z | |
| (code >= 0x0391 && code <= 0x03C9) || // Greek | |
| (code >= 0x0410 && code <= 0x044F) { // Cyrillic | |
| has_ltr = true; | |
| } | |
| } | |
| match (has_rtl, has_ltr) { | |
| (true, false) => "RTL".to_string(), | |
| (false, true) => "LTR".to_string(), | |
| (true, true) => "MIXED".to_string(), | |
| (false, false) => "NEUTRAL".to_string(), | |
| } | |
| } | |
| /// Verify encodeโdecode roundtrip consistency | |
| /// | |
| /// Encodes text, extracts code points, reconstructs, and checks equality | |
| /// Returns true if roundtrip preserves original string | |
| pub fn verify_roundtrip(input: &str) -> bool { | |
| // Method 1: Direct encode/decode via code points | |
| let encoded: String = input | |
| .chars() | |
| .filter_map(|c| char::from_u32(c as u32)) | |
| .collect(); | |
| if encoded != input { | |
| return false; | |
| } | |
| // Method 2: NFC roundtrip | |
| let nfc_form: String = input.nfc().collect(); | |
| let nfc_again: String = nfc_form.nfc().collect(); | |
| if nfc_form != nfc_again { | |
| return false; | |
| } | |
| // Method 3: UTF-8 byte roundtrip | |
| let bytes = input.as_bytes(); | |
| if let Ok(reconstructed) = std::str::from_utf8(bytes) { | |
| reconstructed == input | |
| } else { | |
| false | |
| } | |
| } | |
| mod tests { | |
| use super::*; | |
| fn test_normalize_nfc() { | |
| // รฉ can be represented as single character or e + combining acute | |
| let decomposed = "e\u{0301}"; // e + combining acute | |
| let normalized = normalize(decomposed, "NFC"); | |
| let expected = "รฉ"; // single character | |
| assert_eq!(normalized, expected); | |
| } | |
| fn test_normalize_nfkc() { | |
| let input = "๏ฌ"; // fi ligature U+FB01 | |
| let normalized = normalize(input, "NFKC"); | |
| assert_eq!(normalized, "fi"); | |
| } | |
| fn test_encode_ascii() { | |
| let input = "hello"; | |
| let result = encode(input); | |
| let parsed: Value = serde_json::from_str(&result).unwrap(); | |
| assert_eq!(parsed["normalized"], "hello"); | |
| assert_eq!(parsed["length"], 5); | |
| assert_eq!(parsed["byteLength"], 5); | |
| assert_eq!( | |
| parsed["codePoints"].as_array().unwrap().len(), | |
| 5 | |
| ); | |
| } | |
| fn test_encode_emoji() { | |
| let input = "๐"; // waving hand emoji | |
| let result = encode(input); | |
| let parsed: Value = serde_json::from_str(&result).unwrap(); | |
| assert_eq!(parsed["length"], 1); | |
| assert_eq!(parsed["byteLength"], 4); // UTF-8 encoding of U+1F44B | |
| } | |
| fn test_has_astral_characters_true() { | |
| assert!(has_astral_characters("๐")); // Emoji is astral | |
| assert!(has_astral_characters("๐ณ๐๐๐๐")); // Mathematical alphanumeric symbols | |
| } | |
| fn test_has_astral_characters_false() { | |
| assert!(!has_astral_characters("hello")); | |
| assert!(!has_astral_characters("cafรฉ")); | |
| assert!(!has_astral_characters("ๆฅๆฌ่ช")); // CJK is within BMP | |
| } | |
| fn test_has_combining_marks_true() { | |
| assert!(has_combining_marks("e\u{0301}")); // Explicit combining mark (e + combining acute) | |
| assert!(has_combining_marks("a\u{0308}")); // a with combining diaeresis | |
| } | |
| fn test_has_combining_marks_false() { | |
| assert!(!has_combining_marks("hello")); | |
| assert!(!has_combining_marks("รฉ")); // Precomposed form (NFC) has no combining marks | |
| } | |
| fn test_detect_bidi_ltr() { | |
| let level = detect_bidi_level("Hello World"); | |
| assert_eq!(level, "LTR"); | |
| } | |
| fn test_detect_bidi_rtl() { | |
| let level = detect_bidi_level("ืฉืืื ืขืืื"); // Hebrew | |
| assert_eq!(level, "RTL"); | |
| } | |
| fn test_detect_bidi_mixed() { | |
| let level = detect_bidi_level("Hello ืฉืืื"); | |
| assert_eq!(level, "MIXED"); | |
| } | |
| fn test_detect_bidi_neutral() { | |
| let level = detect_bidi_level("123 !@#"); | |
| assert_eq!(level, "NEUTRAL"); | |
| } | |
| fn test_verify_roundtrip_ascii() { | |
| assert!(verify_roundtrip("hello")); | |
| assert!(verify_roundtrip("The quick brown fox jumps over the lazy dog")); | |
| } | |
| fn test_verify_roundtrip_unicode() { | |
| assert!(verify_roundtrip("cafรฉ")); | |
| assert!(verify_roundtrip("ๆฅๆฌ่ช")); | |
| assert!(verify_roundtrip("๐๐")); | |
| } | |
| fn test_verify_roundtrip_empty() { | |
| assert!(verify_roundtrip("")); | |
| } | |
| fn test_verify_roundtrip_complex() { | |
| // Complex string with multiple scripts and astral characters | |
| let complex = "Hello ู ุฑุญุจุง ืฉืืื ๆฅๆฌ่ช ๐"; | |
| assert!(verify_roundtrip(complex)); | |
| } | |
| } | |