.NET String.Split on EGCs / Text Elements

user_833921 0
Viewed 143

String.split "‍‍‍" "" normally results in ["‍"; "‍‍"], breaking up the visual character.

I want a function available that would instead return ["‍‍‍"], not breaking up the visual character. These "visual characters" are referred to as TextElements in the .NET world, or EGCs (extended grapheme clusters) elsewhere, as far as I'm familiar.

Some details here: https://docs.microsoft.com/en-us/dotnet/api/system.globalization.textelementenumerator?view=net-6.0

Here's a bad solution I've made:

let ecgStringSplit'
  (elements : string list)
  (sep : string list)
  : string list =
  let startsWithSeparator (str : string list) =
    sep = (str |> List.truncate sep.Length)

  let mutable result = []
  let mutable inProgress = []

  let rec recursiveFn (elementsRemaining : string list) : unit =
    if elementsRemaining = [] then
      result <- result @ [ inProgress |> String.concat "" ]
    elif startsWithSeparator elementsRemaining then
      result <- result @ [ inProgress |> String.concat "" ]
      inProgress <- []
      recursiveFn (List.skip sep.Length elementsRemaining)
    else
      inProgress <- inProgress @ [ elementsRemaining.Head ]
      recursiveFn elementsRemaining.Tail

  recursiveFn elements

  result

let toEgcList (s : string) : string list =
  seq {
    let tee = System.Globalization.StringInfo.GetTextElementEnumerator(s)

    while tee.MoveNext() do
      yield tee.GetTextElement()
  } |> Seq.toList

let ecgStringSplit (str: string) (sep: string) =
  ecgStringSplit' (toEgcList str) (toEgcList sep)

ecgStringSplit "Hello world 1" " " |> printfn "%A" // results in ["Hello"; "world"; "1"]
ecgStringSplit "‍‍‍" "" |> printfn "%A" // results in ["‍‍‍"]

Beyond refactoring this to not use mutable values and such, I'm wondering if I'm missing a much simpler solution to this problem?

I'd happily take a C# solution that I can port to F#.

Edit: using something other than F# lists would certainly help perf.

f# .net
2 Answers

I think you have the right idea by creating the sequence expression around GetTextElementEnumerator(). This is what I came up with doing a fold, it appears to work (there is some attempt at handling corner cases by filtering out "" in the results).

open System.Globalization

let toTextElements s =
    seq {
        let tee = StringInfo.GetTextElementEnumerator s
        while tee.MoveNext() do
            yield tee.GetTextElement()
    }

let split (text, separator) =
    let lastWord, list =
        text
        |> toTextElements
        |> Seq.fold (fun (currentWord, currentList) element ->
            if element = separator then
                "", currentWord::currentList
            else
                currentWord + element, currentList)
            ("", [])
    
    lastWord::list
    |> Seq.filter ((<>) "")
    |> Seq.rev
    |> Seq.toList

split ("Hello world 1", " ") // ["Hello"; "world"; "1"]
split ("‍‍‍‍‍‍", "") // ["‍‍‍"; "‍‍‍"]
split ("‍‍‍", "‍‍‍") // [""; ""]
split (" ", " ") // []
user_15942630

A refactored form of the OP

let ecgStringSplit str sep =
  let startsWithSeparator str = sep = (str |> List.truncate sep.Length)

  let result = ResizeArray<string>()

  let rec r (strRemaining : List<string>, inProgress) : unit =
    if strRemaining = [] then
      result |> ResizeArray.append (inProgress.ToString())
    elif startsWithSeparator strRemaining then
      result |> ResizeArray.append (inProgress.ToString())

      r (List.skip sep.Length strRemaining, StringBuilder())
    else
      r (strRemaining.Tail, inProgress.Append(strRemaining.Head))

  r (str, StringBuilder())

  result |> ResizeArray.toList
user_8339210
Related