Apache Ignite 2.10 There is no client connection, but the server heap increases slowly

user_14097092 0
Viewed 98

I have a. Net Core console program as the cache server program. The configuration is described in the following code.

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xmlns:util="http://www.springframework.org/schema/util"
       xsi:schemaLocation="
       http://www.springframework.org/schema/beans
       http://www.springframework.org/schema/beans/spring-beans.xsd
       http://www.springframework.org/schema/util
       http://www.springframework.org/schema/util/spring-util.xsd">
    <bean class="org.apache.ignite.configuration.IgniteConfiguration">
        <property name="includeEventTypes">
            <list>
                <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_STARTED"/>
                <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FINISHED"/>
                <util:constant static-field="org.apache.ignite.events.EventType.EVT_JOB_FINISHED"/>
                <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FAILED"/>
                <util:constant static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_EXPIRED"/>
                <util:constant static-field="org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED"/>
                <util:constant static-field="org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED"/>
                <util:constant static-field="org.apache.ignite.events.EventType.EVT_NODE_FAILED"/>
            </list>
        </property>
        <property name="discoverySpi">
            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
                <property name="ipFinder">
                    <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
                        <property name="addresses">
                            <list>
                                <value>127.0.0.1:47500</value>
                            </list>
                        </property>
                    </bean>
                </property>
                <property name="socketTimeout" value="300" />
                <property name="joinTimeout" value="300" />
            </bean>
        </property>
        <property name="dataStorageConfiguration">
            <bean class="org.apache.ignite.configuration.DataStorageConfiguration">
                <property name="writeThrottlingEnabled" value="true"/>
                <!-- Redefining the default region's settings -->
                <property name="defaultDataRegionConfiguration">
                    <bean class="org.apache.ignite.configuration.DataRegionConfiguration">
                        <property name="name" value="Default_Region"/>
                        <property name="maxSize" value="#{1L * 1024 * 1024 * 1024}"/>
                        <property name="persistenceEnabled" value="true"/>
                        <property name="checkpointPageBufferSize" value="#{8 * 1024 * 1024 * 1024}"/>
                    </bean>
                </property>
                <property name="dataRegionConfigurations">
                    <list>
                        <bean class="org.apache.ignite.configuration.DataRegionConfiguration">
                            <property name="name" value="500MB_Region"/>
                            <property name="initialSize" value="#{100L * 1024 * 1024}"/>
                            <property name="maxSize" value="#{500L * 1024 * 1024}"/>
                            <property name="persistenceEnabled" value="true"/>
                        </bean>
                    </list>
                </property>
                <property name="walMode" value="LOG_ONLY"/>
                <property name="storagePath" value="/data/local/db" />
                <property name="walPath" value="/data/local/db/wal" />
                <property name="walArchivePath" value="/data/local/db/wal/archive" />
            </bean>
        </property>
    </bean>
</beans>

public static IgniteConfiguration GetConfiguration(string springConfigUrl, int jvmInitialMemoryMb, int jvmMaxMemoryMb)
        {
            var igniteHome = IgniteHome.GetIgniteHome();
            var workDir = Path.Combine(igniteHome, "work");

            return new IgniteConfiguration()
            {
                BinaryConfiguration = new BinaryConfiguration
                {
                    Serializer = new BinaryReflectiveSerializer
                    {
                        ForceTimestamp = true
                    }
                },
                SpringConfigUrl = springConfigUrl,
                AutoGenerateIgniteInstanceName = true,
                JvmInitialMemoryMb = jvmInitialMemoryMb,
                JvmMaxMemoryMb = jvmMaxMemoryMb,
                LifecycleHandlers = new[] { LifecycleAwareExample },
                FailureHandler = new StopNodeFailureHandler(),
                ClientFailureDetectionTimeout = TimeSpan.FromSeconds(5),
                FailureDetectionTimeout = TimeSpan.FromSeconds(60),
                IgniteHome = igniteHome,
                WorkDirectory = workDir,             
                SystemWorkerBlockedTimeout = TimeSpan.FromSeconds(10),
                JvmOptions = JvmOpts,
                CommunicationSpi = new TcpCommunicationSpi
                {
                    ConnectTimeout = TimeSpan.FromMilliseconds(5000),
                    MaxConnectTimeout = TimeSpan.FromMilliseconds(10000)
                },

            };
        }

        private static readonly IList<string> JvmOpts = Environment.Is64BitProcess
       ? new List<string>
       {
                    "-XX:+HeapDumpOnOutOfMemoryError",
                    "-DIGNITE_QUIET=true",
                    "-Duser.timezone=UTC",
                    "-Djava.net.preferIPv4Stack=true"
       }
       : new List<string>
       {
                    "-XX:+HeapDumpOnOutOfMemoryError",
                    "-DIGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE=1000",
                    "-DIGNITE_QUIET=true",
                    "-Duser.timezone=UTC",
                    "-Djava.net.preferIPv4Stack=true"
       };

I start the ignite method and the JVM settings

var server = Ignition.Start(GetConfiguration("IgniteConfig\\config.xml", 1024, 2048));

After I started the server program, I found that I didn't do anything, and the memory usage of running my program for a long time was also growing slowly. Has anyone ever seen this kind of slow growth of memory without doing anything? Or I still lack some configuration.

The console print section is shown below：

[08:17:16,984][INFO][grid-timeout-worker-#22%ignite-instance-1e46f40f-14be-4af5-a5ea-10ccfc8fbbd7%][IgniteKernal%ignite-instance-1e46f40f-14be-4af5-a5ea-10ccfc8fbbd7]
Metrics for local node (to disable set 'metricsLogFrequency' to 0)
    ^-- Node [id=cd7d5ca5, name=ignite-instance-1e46f40f-14be-4af5-a5ea-10ccfc8fbbd7, uptime=00:25:00.979]
    ^-- Cluster [hosts=1, CPUs=6, servers=1, clients=0, topVer=1, minorTopVer=1]
    ^-- Network [addrs=[127.0.0.1, 192.168.50.135], discoPort=47500, commPort=47100]
    ^-- CPU [CPUs=6, curLoad=0%, avgLoad=0.02%, GC=0%]
    ^-- Heap [used=166MB, free=91.87%, comm=1024MB]
    ^-- Off-heap memory [used=0MB, free=99.99%, allocated=199MB]
    ^-- Page memory [pages=25]
    ^--   sysMemPlc region [type=internal, persistence=true, lazyAlloc=false,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=99.99%, allocRam=99MB, allocTotal=0MB]
    ^--   metastoreMemPlc region [type=internal, persistence=true, lazyAlloc=false,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=99.91%, allocRam=0MB, allocTotal=0MB]
    ^--   500MB_Region region [type=user, persistence=true, lazyAlloc=true,
      ...  initCfg=100MB, maxCfg=500MB, usedRam=0MB, freeRam=100%, allocRam=0MB, allocTotal=0MB]
    ^--   TxLog region [type=internal, persistence=true, lazyAlloc=false,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=99MB, allocTotal=0MB]
    ^--   volatileDsMemPlc region [type=user, persistence=false, lazyAlloc=true,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=0MB]
    ^--   Default_Region region [type=default, persistence=true, lazyAlloc=true,
      ...  initCfg=256MB, maxCfg=1024MB, usedRam=0MB, freeRam=100%, allocRam=0MB, allocTotal=0MB]
    ^-- Ignite persistence [used=0MB]
    ^-- Outbound messages queue [size=0]
    ^-- Public thread pool [active=0, idle=0, qSize=0]
    ^-- System thread pool [active=0, idle=7, qSize=0]
[08:18:17,018][INFO][grid-timeout-worker-#22%ignite-instance-1e46f40f-14be-4af5-a5ea-10ccfc8fbbd7%][IgniteKernal%ignite-instance-1e46f40f-14be-4af5-a5ea-10ccfc8fbbd7]
Metrics for local node (to disable set 'metricsLogFrequency' to 0)
    ^-- Node [id=cd7d5ca5, name=ignite-instance-1e46f40f-14be-4af5-a5ea-10ccfc8fbbd7, uptime=00:26:01.019]
    ^-- Cluster [hosts=1, CPUs=6, servers=1, clients=0, topVer=1, minorTopVer=1]
    ^-- Network [addrs=[127.0.0.1, 192.168.50.135], discoPort=47500, commPort=47100]
    ^-- CPU [CPUs=6, curLoad=0%, avgLoad=0.02%, GC=0%]
    ^-- Heap [used=169MB, free=91.72%, comm=1024MB]
    ^-- Off-heap memory [used=0MB, free=99.99%, allocated=199MB]
    ^-- Page memory [pages=25]
    ^--   sysMemPlc region [type=internal, persistence=true, lazyAlloc=false,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=99.99%, allocRam=99MB, allocTotal=0MB]
    ^--   metastoreMemPlc region [type=internal, persistence=true, lazyAlloc=false,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=99.91%, allocRam=0MB, allocTotal=0MB]
    ^--   500MB_Region region [type=user, persistence=true, lazyAlloc=true,
      ...  initCfg=100MB, maxCfg=500MB, usedRam=0MB, freeRam=100%, allocRam=0MB, allocTotal=0MB]
    ^--   TxLog region [type=internal, persistence=true, lazyAlloc=false,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=99MB, allocTotal=0MB]
    ^--   volatileDsMemPlc region [type=user, persistence=false, lazyAlloc=true,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=0MB]
    ^--   Default_Region region [type=default, persistence=true, lazyAlloc=true,
      ...  initCfg=256MB, maxCfg=1024MB, usedRam=0MB, freeRam=100%, allocRam=0MB, allocTotal=0MB]
    ^-- Ignite persistence [used=0MB]
    ^-- Outbound messages queue [size=0]
    ^-- Public thread pool [active=0, idle=0, qSize=0]
    ^-- System thread pool [active=0, idle=7, qSize=0]
[08:19:16,163][INFO][db-checkpoint-thread-#65%ignite-instance-1e46f40f-14be-4af5-a5ea-10ccfc8fbbd7%][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, reason='timeout']
[08:19:17,047][INFO][grid-timeout-worker-#22%ignite-instance-1e46f40f-14be-4af5-a5ea-10ccfc8fbbd7%][IgniteKernal%ignite-instance-1e46f40f-14be-4af5-a5ea-10ccfc8fbbd7]
Metrics for local node (to disable set 'metricsLogFrequency' to 0)
    ^-- Node [id=cd7d5ca5, name=ignite-instance-1e46f40f-14be-4af5-a5ea-10ccfc8fbbd7, uptime=00:27:01.047]
    ^-- Cluster [hosts=1, CPUs=6, servers=1, clients=0, topVer=1, minorTopVer=1]
    ^-- Network [addrs=[127.0.0.1, 192.168.50.135], discoPort=47500, commPort=47100]
    ^-- CPU [CPUs=6, curLoad=0%, avgLoad=0.02%, GC=0%]
    ^-- Heap [used=173MB, free=91.53%, comm=1024MB]
    ^-- Off-heap memory [used=0MB, free=99.99%, allocated=199MB]
    ^-- Page memory [pages=25]
    ^--   sysMemPlc region [type=internal, persistence=true, lazyAlloc=false,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=99.99%, allocRam=99MB, allocTotal=0MB]
    ^--   metastoreMemPlc region [type=internal, persistence=true, lazyAlloc=false,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=99.91%, allocRam=0MB, allocTotal=0MB]
    ^--   500MB_Region region [type=user, persistence=true, lazyAlloc=true,
      ...  initCfg=100MB, maxCfg=500MB, usedRam=0MB, freeRam=100%, allocRam=0MB, allocTotal=0MB]
    ^--   TxLog region [type=internal, persistence=true, lazyAlloc=false,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=99MB, allocTotal=0MB]
    ^--   volatileDsMemPlc region [type=user, persistence=false, lazyAlloc=true,
      ...  initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=0MB]
    ^--   Default_Region region [type=default, persistence=true, lazyAlloc=true,
      ...  initCfg=256MB, maxCfg=1024MB, usedRam=0MB, freeRam=100%, allocRam=0MB, allocTotal=0MB]
    ^-- Ignite persistence [used=0MB]
    ^-- Outbound messages queue [size=0]
    ^-- Public thread pool [active=0, idle=0, qSize=0]
    ^-- System thread pool [active=0, idle=7, qSize=0]
ignite asp.net-core .net
1 Answers

Any Java application which does anything is bound to increase heap use until Full GC is initiated. Current heap usage is just not an informative metric for determining whether your real memory usage is growing or stable. Consider collecting GC logs, but you will still need to cycle through a few Full GCs to be able to answer that question.

user_364980
Related