Hello fellow developers!
When developing a utility application for Mac OS, I encountered an incomprehensible memory leak effect, presumably when working with String type data. I will briefly describe the essence:
For my main application, I need some data embedded directly into the assembly. These data need to be taken according to certain criteria from a very massive, I would even say "heavy as the sins of all mankind", XML file type. The file is incredibly large. The volume exceeds 1.7 TB (i.e. 1700 gigabytes).
The XMLParser() provided by Apple freezes at around 57 million lines. At the same time, the amount of RAM used by the application stops at 7-7.2 gigabytes with a physical limit of 8 gigabytes (my computer has an M1 chip and a RAM volume of 8 gigabytes). Therefore, it was decided to create a service application (without any frills like a beautiful interface) that performs only the task I need.
And here's a surprise! I ran into an even more interesting problem: the application works as much as possible, but about the third or fourth gigabyte of the file, the amount of memory used grows to 12-13 gigabytes (let me remind you about the physical limit of 8 gigabytes). It is clear that all the excess is merged into the swap file. But that doesn't make it any easier. My patience lasted up to 20 gigabytes, then I completed the application. Maybe someone is interested - he will look and be able to figure out what the problem is. My brain is already broken. Here is the source code:
import SwiftUI
import UniformTypeIdentifiers
struct ContentView: View {
@State var showImporter: Bool = false
@State var stream: InputStream?
@State var logs: [String] = []
@State var nodeMaxID: Int = 0
let queue = DispatchQueue.init(label: "Parser", qos: .userInteractive, attributes: .concurrent, autoreleaseFrequency: .inherit, target: nil)
var body: some View {
VStack(alignment: .leading, spacing: 4) {
HStack(){
Button(action: {showImporter = true}) { Text("Open file") }
Spacer()
}
Text("Node max ID: \(nodeMaxID)")
ScrollViewReader() { proxy in
ScrollView(.vertical, showsIndicators: true) {
VStack(alignment: .leading, spacing: 4) {
ForEach(logs, id: \.self) { log in
Text(log)
.id(log)
.onAppear{ withAnimation { proxy.scrollTo(log) } }
}
}
}
}
}
.padding()
.frame(minWidth: 640, idealWidth: 1280, maxWidth: 1920, minHeight: 360, idealHeight: 720, maxHeight: 1080, alignment: .center)
.fileImporter(isPresented: $showImporter, allowedContentTypes: [UTType.data]) { result in
do {
let url = try result.get()
self.stream = InputStream(url: url)
}
catch { print(error.localizedDescription) }
}
.onChange(of: stream) { _ in
if let stream = stream { parseData(from: stream) }
}
}
var elements: [String] = []
func parseData(from inputStream: InputStream) {
queue.asyncAfter(deadline: .now() + .seconds(1)) {
var count = 0
let l = 128*1024*1024
let buf = UnsafeMutablePointer<UInt8>.allocate(capacity: l)
var lastLine = ""
var str = ""
var lines: [String] = []
inputStream.open()
while inputStream.hasBytesAvailable {
logs.append("[\(count)] Start read data to buffer")
let numberOfBytesRead = inputStream.read(buf, maxLength: l)
if numberOfBytesRead < 0, let error = inputStream.streamError { print(error); break }
logs.append("[\(count)] Start converting data to string")
str = String(bytesNoCopy: buf, length: l, encoding: .utf8, freeWhenDone: false) ?? ""
logs.append("[\(count)] Start separating string to lines")
lines = (lastLine + str).components(separatedBy: "\n").filter{$0.count > 3}
lastLine = lines.removeLast()
logs.append("[\(count)] String buffer count: \(str.count), Lines count: \(lines.count)")
while !lines.isEmpty {
var line = lines.removeFirst()
while line.first ?? "@" == " " { line.removeFirst() }
if line.first ?? "@" == "<" { line.removeFirst() }
if line.last ?? "@" == ">" { line.removeLast() }
var name = ""
//print(line)
while line.first ?? "@" != " " && !line.isEmpty { name.append(line.removeFirst())}
if line.first ?? "@" == " " { line.removeFirst() }
let array = line.components(separatedBy: "\" ")
if array.count == 1 && name.first ?? "@" == "/" {
name.removeFirst()
didEndElement(name)
} else {
var attributes: [String: String] = [:]
for item in array {
var dic = item.components(separatedBy: "=\"")
if dic.count == 2 {
if dic[1].last ?? "@" == "/" { dic[1].removeLast() }
if dic[1].last ?? "@" == "\"" { dic[1].removeLast() }
attributes[dic[0]] = dic[1]
}
}
didStartElement(name, attributes: attributes)
if line.last ?? "@" == "/" { didEndElement(name) }
}
}
str = ""
count += 1
}
inputStream.close()
}
}
func didStartElement(_ name: String, attributes: [String: String]) {
}
func didEndElement(_ name: String) {
}
}
Thank you in advance for any help.