We recently introduced ElasticSearch and are embedding it in our automated testing framework using pytest and testcontainers-python. Opon execution of our tests I now experience a high performance loss, mainly due to start up time of the ES-Container. Whil PostgreSQL-Containers take less than a second to be ready, I measure ~15 seconds for an ES-Container to be queryable.
Is there a way to increase boot time, e.g. by providing certain start up flags for disabling checks only needed for productive use?
The host system is macOS Catalina (10.15.5) with docker-machine installed through macOS docker-toolbox.
The script I use to start the container is as follows:
docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" --detach=true --name=elasticsearch docker.elastic.co/elasticsearch/elasticsearch:7.8.1
Here are the resulting logs (I had to strip away the adding index template and adding template rows due to StackOverflow character limitations):
{"type": "server", "timestamp": "2020-08-14T12:46:12,454Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "version[7.8.1], pid[6], build[default/docker/b5ca9c58fb664ca8bf9e4057fc229b3396bf3a89/2020-07-21T16:40:44.668009Z], OS[Linux/4.14.154-boot2docker/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/14.0.1/14.0.1+7]" }
{"type": "server", "timestamp": "2020-08-14T12:46:12,466Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "JVM home [/usr/share/elasticsearch/jdk]" }
{"type": "server", "timestamp": "2020-08-14T12:46:12,467Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, -Xms1g, -Xmx1g, -XX:+UseG1GC, -XX:G1ReservePercent=25, -XX:InitiatingHeapOccupancyPercent=30, -Djava.io.tmpdir=/tmp/elasticsearch-15659003528260691505, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -XX:MaxDirectMemorySize=536870912, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,571Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,574Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,574Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,575Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [flattened]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,577Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,578Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,580Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,580Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,580Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,582Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,583Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,583Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,584Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,584Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,584Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,586Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,587Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,587Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,592Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,592Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,592Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,593Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [tasks]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,594Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,595Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,596Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,596Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,597Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,597Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,597Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,598Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,598Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,599Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,599Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,600Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,601Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,602Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,603Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,603Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,604Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,604Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,606Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,607Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,607Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,609Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,609Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,611Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "no plugins loaded" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,719Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "using [1] data paths, mounts [[/ (overlay)]], net usable_space [35.3gb], net total_space [46.1gb], types [overlay]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,719Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "heap size [1gb], compressed ordinary object pointers [true]" }
{"type": "server", "timestamp": "2020-08-14T12:46:16,800Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "node name [5a13d8aa092e], node ID [1mTfit4SQ_-kQshQRSjHUA], cluster name [docker-cluster]" }
{"type": "server", "timestamp": "2020-08-14T12:46:24,901Z", "level": "INFO", "component": "o.e.x.s.a.s.FileRolesStore", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]" }
{"type": "server", "timestamp": "2020-08-14T12:46:26,095Z", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "[controller/167] [Main.cc@115] controller (64 bit): Version 7.8.1 (Build d0d3f60f03220d) Copyright (c) 2020 Elasticsearch BV" }
{"type": "server", "timestamp": "2020-08-14T12:46:27,488Z", "level": "DEBUG", "component": "o.e.a.ActionModule", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "Using REST wrapper from plugin org.elasticsearch.xpack.security.Security" }
{"type": "server", "timestamp": "2020-08-14T12:46:27,759Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "using discovery type [single-node] and seed hosts providers [settings]" }
{"type": "server", "timestamp": "2020-08-14T12:46:29,664Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "initialized" }
{"type": "server", "timestamp": "2020-08-14T12:46:29,669Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "starting ..." }
{"type": "server", "timestamp": "2020-08-14T12:46:29,887Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "publish_address {172.17.0.5:9300}, bound_addresses {0.0.0.0:9300}" }
{"type": "server", "timestamp": "2020-08-14T12:46:30,167Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "setting initial configuration to VotingConfiguration{1mTfit4SQ_-kQshQRSjHUA}" }
{"type": "server", "timestamp": "2020-08-14T12:46:30,377Z", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "elected-as-master ([1] nodes joined)[{5a13d8aa092e}{1mTfit4SQ_-kQshQRSjHUA}{uOmlUgNfSGGdKHmVT1gjAw}{172.17.0.5}{172.17.0.5:9300}{dilmrt}{ml.machine_memory=8368738304, xpack.installed=true, transform.node=true, ml.max_open_jobs=20} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 1, version: 1, delta: master node changed {previous [], current [{5a13d8aa092e}{1mTfit4SQ_-kQshQRSjHUA}{uOmlUgNfSGGdKHmVT1gjAw}{172.17.0.5}{172.17.0.5:9300}{dilmrt}{ml.machine_memory=8368738304, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}]}" }
{"type": "server", "timestamp": "2020-08-14T12:46:30,452Z", "level": "INFO", "component": "o.e.c.c.CoordinationState", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "cluster UUID set to [34xpmI2KQOGWkn8QmXY4Gw]" }
{"type": "server", "timestamp": "2020-08-14T12:46:30,492Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "master node changed {previous [], current [{5a13d8aa092e}{1mTfit4SQ_-kQshQRSjHUA}{uOmlUgNfSGGdKHmVT1gjAw}{172.17.0.5}{172.17.0.5:9300}{dilmrt}{ml.machine_memory=8368738304, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}]}, term: 1, version: 1, reason: Publication{term=1, version=1}" }
{"type": "server", "timestamp": "2020-08-14T12:46:30,591Z", "level": "INFO", "component": "o.e.h.AbstractHttpServerTransport", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "publish_address {172.17.0.5:9200}, bound_addresses {0.0.0.0:9200}", "cluster.uuid": "34xpmI2KQOGWkn8QmXY4Gw", "node.id": "1mTfit4SQ_-kQshQRSjHUA" }
{"type": "server", "timestamp": "2020-08-14T12:46:30,592Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "started", "cluster.uuid": "34xpmI2KQOGWkn8QmXY4Gw", "node.id": "1mTfit4SQ_-kQshQRSjHUA" }
{"type": "server", "timestamp": "2020-08-14T12:46:30,797Z", "level": "INFO", "component": "o.e.g.GatewayService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "recovered [0] indices into cluster_state", "cluster.uuid": "34xpmI2KQOGWkn8QmXY4Gw", "node.id": "1mTfit4SQ_-kQshQRSjHUA" }
{"type": "server", "timestamp": "2020-08-14T12:46:32,508Z", "level": "INFO", "component": "o.e.x.i.a.TransportPutLifecycleAction", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "adding index lifecycle policy [watch-history-ilm-policy]", "cluster.uuid": "34xpmI2KQOGWkn8QmXY4Gw", "node.id": "1mTfit4SQ_-kQshQRSjHUA" }
{"type": "server", "timestamp": "2020-08-14T12:46:32,576Z", "level": "INFO", "component": "o.e.x.i.a.TransportPutLifecycleAction", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "adding index lifecycle policy [ml-size-based-ilm-policy]", "cluster.uuid": "34xpmI2KQOGWkn8QmXY4Gw", "node.id": "1mTfit4SQ_-kQshQRSjHUA" }
{"type": "server", "timestamp": "2020-08-14T12:46:32,643Z", "level": "INFO", "component": "o.e.x.i.a.TransportPutLifecycleAction", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "adding index lifecycle policy [ilm-history-ilm-policy]", "cluster.uuid": "34xpmI2KQOGWkn8QmXY4Gw", "node.id": "1mTfit4SQ_-kQshQRSjHUA" }
{"type": "server", "timestamp": "2020-08-14T12:46:32,694Z", "level": "INFO", "component": "o.e.x.i.a.TransportPutLifecycleAction", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "adding index lifecycle policy [slm-history-ilm-policy]", "cluster.uuid": "34xpmI2KQOGWkn8QmXY4Gw", "node.id": "1mTfit4SQ_-kQshQRSjHUA" }
{"type": "server", "timestamp": "2020-08-14T12:46:32,893Z", "level": "INFO", "component": "o.e.l.LicenseService", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "license [f666aafd-836c-4890-85ea-bdb81cc6617f] mode [basic] - valid", "cluster.uuid": "34xpmI2KQOGWkn8QmXY4Gw", "node.id": "1mTfit4SQ_-kQshQRSjHUA" }
{"type": "server", "timestamp": "2020-08-14T12:46:32,895Z", "level": "INFO", "component": "o.e.x.s.s.SecurityStatusChangeListener", "cluster.name": "docker-cluster", "node.name": "5a13d8aa092e", "message": "Active license is now [BASIC]; Security is disabled", "cluster.uuid": "34xpmI2KQOGWkn8QmXY4Gw", "node.id": "1mTfit4SQ_-kQshQRSjHUA" }