Need help fixing memory leak for elixir and sweet_xml library

Viewed 477

I'm a newbie in elixir.

I've got the following lib/osm.ex file

defmodule Osm do
  import SweetXml

  def hello do
    :world
  end

  def main(args) do
    args |> parse_args |> process
  end

  defp parse_args(args) do
    {options, _, _} = OptionParser.parse(args, switches: [osm_file: :string, help: :boolean])
    options
  end

  def output_help() do
    IO.puts "Usage: osm [OPTION]"
    IO.puts ""
    IO.puts "  --osm-file  an osm-file to import"
    IO.puts "  --help      outputs this help-page"
  end

  def process([]) do
    IO.puts "No arguments given"
  end

  def process(options) do
    if options[:help] do
      output_help()
    else
      case options do
        [osm_file: _] ->
          process_osm_file(options[:osm_file])
      end
    end
  end

  def process_osm_file(file) do
    counts = %{:nodes => 0, :ways => 0, :relations => 0}
    cond do
      String.ends_with?(file, ".pbf") ->
        IO.puts "parse osm-pbf."
      String.ends_with?(file, ".osm.bz2") ->
        IO.puts "extract and parse osm-xml."
      String.ends_with?(file, ".osm") ->
        IO.puts "parse osm-xml."
        File.stream!(file)
         |> stream_tags([:node, :way, :relation], discard: [:node, :way, :relation])
          |> Stream.map(fn
            {_, node} ->
              process_element(node, counts)
          end)
          |> Enum.reduce(fn element, result ->
            result_modified = %{result |
              nodes: result[:nodes] + element[:nodes],
              ways: result[:ways] + element[:ways],
              relations: result[:relations] + element[:relations]
            }
            cond do
              rem(result_modified[:nodes], 1000) == 0 ->
                IO.write "\rnodes: " <> to_string(result_modified[:nodes]) <> "; ways: " <> to_string(result_modified[:ways]) <> "; relations: " <> to_string(result_modified[:relations]) <> "; mem: " <> to_string(:erlang.memory(:total))
              true -> true
            end
            result_modified
          end)
          |> Stream.run
          IO.puts ""
      true ->
        IO.puts "invalid osm-file extension."
    end
  end

  defp process_element(doc, counts) do
    case doc |> xmlElement(:name) do
      :node ->
        doc |> xmap(
          id: ~x"./@id"i,
          lat: ~x"./@lat"f,
          lon: ~x"./@lon"f,
          tags: [
            ~x"./tag"l,
            key: ~x"./@k"s,
            value: ~x"./@v"s
          ]
        ) |> process_node(counts)

      :way ->
        doc |> xmap(
          id: ~x"./@id"i,
          nd: [
            ~x"./nd"l,
            ref: ~x"./@ref"i
          ],
          tags: [
            ~x"./tag"l,
            key: ~x"./@k"s,
            value: ~x"./@v"s
          ]
        ) |> process_way(counts)

      :relation ->
        doc |> xmap(
          id: ~x"./@id"i,
          member: [
            ~x"./member"l,
            type: ~x"./@type"s,
            ref: ~x"./@ref"s,
            role: ~x"./@role"s
          ],
          tags: [
            ~x"./tag"l,
            key: ~x"./@k"s,
            value: ~x"./@v"s
          ]
        ) |> process_relation(counts)

      _ ->
        IO.puts "unhandled element"
    end
  end

  defp process_node(node, counts) do
    _ = node
    Map.put(counts, :nodes, counts[:nodes] + 1)
  end

  defp process_way(way, counts) do
    _ = way
    Map.put(counts, :ways, counts[:ways] + 1)
  end

  defp process_relation(relation, counts) do
    _ = relation
    Map.put(counts, :relations, counts[:relations] + 1)
  end
end

and the following mix.exs file

defmodule Osm.MixProject do
  use Mix.Project

  def project do
    [
      app: :osm,
      version: "0.1.0",
      elixir: "~> 1.7",
      start_permanent: Mix.env() == :prod,
      escript: [main_module: Osm],
      deps: deps()
    ]
  end

  def application do
    [
      extra_applications: [:logger]
    ]
  end

  defp deps do
    [
      {:sweet_xml, github: 'kbrw/sweet_xml', app: false}
    ]
  end
end

I compile it with mix escript.build

I've downloaded the berlin-latest.osm.bz2 file and extracted the berlin-latest.osm file.

If I call ./osm --osm-file=berlin-latest.osm

The script parses the xml-data and counts the nodes, ways and relations correctly, but the memory consumption is increasing until the end.

Is there a memory leak in the SweetXml library or am I doing something wrong?

1 Answers

I do not see something that would create a memory leak in your code.

I did the following test: I removed progressively all the code using SweetXml, and it is when I withdrew the first part using SweetXml (i.e: stream_tags([:node, :way, :relation], discard: [:node, :way, :relation])) that the memory leak disappeared. This clearly indicates that the memory consumption comes from SweetXml

Reading the source code of SweetXml.stream_tags/3 function, might bring you some answers. I haven't yet understood from where the leak comes from.

Edit: after a thorough inspection of the source code, I still haven't found the origin of the leak. I am starting to thing it is something even deeper, maybe linked to the way erlang VM works.

Related