How to concatenate Apache Arrow files with identical structure in julia

Viewed 205

How can I concatenate several Arrow files with identical structure into a single Arrow file without reading each file into memory? I am using Arrow.jl and the Arrow files represent dataframes with identical structure and the combined dataframes are larger than available memory. I have been doing the following:

function openhhmembershipchunk(basedirectory::String, node::String, chunk::Int64)
    open(joinpath(basedirectory, node, "DayExtraction", "HouseholdMembershipDays$(chunk).arrow")) do io
        return Arrow.Table(io) |> DataFrame
    end;
end
"Concatenate household membership day batches"
function combinehhmembershipdaybatch(basedirectory::String, node::String, batches)
    d = openhhmembershipchunk(basedirectory::String, node::String, 1)
    days = copy(d)
    for i = 2:batches
        m = openhhmembershipchunk(basedirectory::String, node::String, i)
        append!(days, m)
    end
    Arrow.write(joinpath(basedirectory, node, "DayExtraction", "HouseholdMembershipDays.arrow"), days, compress=:zstd)
    #delete chunks
    for i = 1:batches
        rm(joinpath(basedirectory, node, "DayExtraction", "HouseholdMembershipDays$(i).arrow"))
    end
    return nothing
end #combinehhresidencydaybatch
0 Answers
Related