How can I concatenate several Arrow files with identical structure into a single Arrow file without reading each file into memory? I am using Arrow.jl and the Arrow files represent dataframes with identical structure and the combined dataframes are larger than available memory. I have been doing the following:
function openhhmembershipchunk(basedirectory::String, node::String, chunk::Int64)
open(joinpath(basedirectory, node, "DayExtraction", "HouseholdMembershipDays$(chunk).arrow")) do io
return Arrow.Table(io) |> DataFrame
end;
end
"Concatenate household membership day batches"
function combinehhmembershipdaybatch(basedirectory::String, node::String, batches)
d = openhhmembershipchunk(basedirectory::String, node::String, 1)
days = copy(d)
for i = 2:batches
m = openhhmembershipchunk(basedirectory::String, node::String, i)
append!(days, m)
end
Arrow.write(joinpath(basedirectory, node, "DayExtraction", "HouseholdMembershipDays.arrow"), days, compress=:zstd)
#delete chunks
for i = 1:batches
rm(joinpath(basedirectory, node, "DayExtraction", "HouseholdMembershipDays$(i).arrow"))
end
return nothing
end #combinehhresidencydaybatch