From this reference:
It states that the value of string.length is actually the number of UTF-16 code units, and not the number of characters.
I am apparently naive in assuming that any UTF-8 characters 3 or 4 bytes wide, would have to take up 2 UTF-16 code units. Here is what I mean:
I did a little experimenting with the string: œ´®†¥¨ˆøπ¬˚∆˙©ƒ∂ßåΩ≈ç√∫˜µ≤ユーザーコードa which contains a mix of 1, 2, 3 and 4 byte wide characters. I got some surprising results.
Each UTF-16 code unit is 2 bytes wide. The number of characters in the string is 35. Taking string.length of the string equals 36, meaning only one of the characters took 2 UTF-16 code units, yet there are several UTF-8 characters that are 3 and 4 bytes wide.
Using the code below, I examined each UTF-8 character, the number of bytes it uses, and the string.length of it. It was interesting to me that all of the 3-byte UTF-8 characters only used a single UTF-16 code unit. The only character requiring 2 code units is the 4-byte wide emoticon.
Can someone please explain how this can be? Thanks!
Code:
function iterateCharacters(str) {
let te = new TextEncoder();
let totalBytes = 0;
let totalCodeUnits1 = 0;
let totalCodeUnits2 = 0;
let arr = [...str];
for (let i = 0; i < arr.length; i++) {
let bytes = te.encode(arr[i]).length;
let length = arr[i].length;
totalBytes += bytes;
console.log(" i: " + i + " char: " + arr[i] + " bytes: " + bytes + " length: " + length);
// Erroneous assumption that more than 2 utf8 bytes would occupy 2 UTF-16 code units:
totalCodeUnits1 += bytes < 3 ? 1 : 2;
totalCodeUnits2 += length;
}
console.log(" total UTF-16 code units (erroneous calculation): " + totalCodeUnits1)
console.log(" total UTF-16 code units (correct calculation): " + totalCodeUnits2)
console.log(" total characters: " + arr.length)
console.log(" total UTF-8 bytes: " + totalBytes)
}
var sample = "œ´®†¥¨ˆøπ¬˚∆˙©ƒ∂ßåΩ≈ç√∫˜µ≤ユーザーコードa";
iterateCharacters(sample);
console.log("total number of UTF-16 code units: " + sample.length);
console.log("total number of characters: " + [...sample].length);
console.log("total number of UTF-8 bytes: " + (new TextEncoder().encode(sample)).length);
The results:
i: 0 char: bytes: 4 length: 2
i: 1 char: œ bytes: 2 length: 1
i: 2 char: ´ bytes: 2 length: 1
i: 3 char: ® bytes: 2 length: 1
i: 4 char: † bytes: 3 length: 1
i: 5 char: ¥ bytes: 2 length: 1
i: 6 char: ¨ bytes: 2 length: 1
i: 7 char: ˆ bytes: 2 length: 1
i: 8 char: ø bytes: 2 length: 1
i: 9 char: π bytes: 2 length: 1
i: 10 char: ¬ bytes: 2 length: 1
i: 11 char: ˚ bytes: 2 length: 1
i: 12 char: ∆ bytes: 3 length: 1
i: 13 char: ˙ bytes: 2 length: 1
i: 14 char: © bytes: 2 length: 1
i: 15 char: ƒ bytes: 2 length: 1
i: 16 char: ∂ bytes: 3 length: 1
i: 17 char: ß bytes: 2 length: 1
i: 18 char: å bytes: 2 length: 1
i: 19 char: Ω bytes: 2 length: 1
i: 20 char: ≈ bytes: 3 length: 1
i: 21 char: ç bytes: 2 length: 1
i: 22 char: √ bytes: 3 length: 1
i: 23 char: ∫ bytes: 3 length: 1
i: 24 char: ˜ bytes: 2 length: 1
i: 25 char: µ bytes: 2 length: 1
i: 26 char: ≤ bytes: 3 length: 1
i: 27 char: ユ bytes: 3 length: 1
i: 28 char: ー bytes: 3 length: 1
i: 29 char: ザ bytes: 3 length: 1
i: 30 char: ー bytes: 3 length: 1
i: 31 char: コ bytes: 3 length: 1
i: 32 char: ー bytes: 3 length: 1
i: 33 char: ド bytes: 3 length: 1
i: 34 char: a bytes: 1 length: 1
total UTF-16 code units (erroneous calculation): 50
total UTF-16 code units (correct calculation): 36
total characters: 35
total UTF-8 bytes: 85
total number of UTF-16 code units: 36
total number of characters: 35
total number of UTF-8 bytes: 85
(See also Jsfiddle: https://jsfiddle.net/Allasso/o5zpmrc9/)