I have a docker-compose with two containers, one with python and other one with Elasticsearch. I'm trying to bulk a lot of txt files in the Elasticsearch DB reading it and inserting it line per line, but for some reason the Elasticsearch machine it stops during the bulk and the connection is refused.
This is the error I'm getting:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/urllib3/connection.py", line 160, in _new_conn
(self._dns_host, self.port), self.timeout, **extra_kw
File "/usr/local/lib/python3.7/site-packages/urllib3/util/connection.py", line 84, in create_connection
raise err
File "/usr/local/lib/python3.7/site-packages/urllib3/util/connection.py", line 74, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
This is my docker-compose file:
version: "3.7"
services:
web:
container_name: web
restart: "always"
build: "./web"
hostname: web
command: python manage.py runserver 0.0.0.0:8000
ports:
- 8000:8000
depends_on:
- els
volumes:
- ./web:/web
- ./data:/data
networks:
- esnet
els:
container_name: els
build: "./containers/elasticsearch"
restart: "always"
hostname: els
ports:
- 9200:9200
- 9300:9300
volumes:
- "./data/elasticsearch:/usr/share/elasticsearch/data"
ulimits:
nproc: 3000
nofile: 65536
memlock: -1
environment:
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx8G"
- discovery.type=single-node
- xpack.security.enabled=false
networks:
- esnet
volumes:
web:
data:
networks:
esnet:
This is the elasticsearch dockerfile:
FROM docker.elastic.co/elasticsearch/elasticsearch:7.2.1
ADD ./elasticsearch.yml /usr/share/elasticsearch/config
EXPOSE 9200 9300
And this is the config:
http.host: 0.0.0.0
http.cors.allow-origin: "*"
http.cors.enabled: true
http.cors.allow-headers: "X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization"
http.cors.allow-methods: "OPTIONS, HEAD, GET, POST, PUT, DELETE"
http.cors.allow-credentials: true
This is my data bulk code:
bulk_list = []
with open(file) as f:
lines = f.readlines()
for line in lines:
l = line.strip()
l = l.replace("\n", "")
if l:
bulk_list.append({"...some data..."})
helpers.bulk(els, bulk_list)
And this is my http://localhost:9200/_nodes/thread_pool?pretty write param:
"write" : {
"type" : "fixed",
"size" : 8,
"queue_size" : 200
}
I think the latter is the problem, but I'm not too sure what is the best way to fix this avoiding the connection refused.
Thanks!
ADDED:
This is the log error in docker-compose:
{"type": "server", "timestamp": "2020-05-24T20:04:13,927Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "elasticsearch", "node.name": "els", "message": "fatal error in thread [elasticsearch[els][refresh][T#1]], exiting", "cluster.uuid": "Ic_DMuXJTQqa4zu3wSEobw", "node.id": "12SGfI1qQkyZ-KJjF_okPA" ,
"stacktrace": ["java.lang.InternalError: a fault occurred in a recent unsafe memory access operation in compiled Java code",
"at org.apache.lucene.index.ByteSliceReader.readByte(ByteSliceReader.java:73) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.store.DataInput.readVInt(DataInput.java:126) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.FreqProxFields$FreqProxPostingsEnum.nextPosition(FreqProxFields.java:501) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:150) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:937) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:347) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3234) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:835) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.IndexService.access$200(IndexService.java:109) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:967) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) ~[?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) ~[?:?]",
"at java.lang.Thread.run(Thread.java:832) [?:?]"] }
{"type": "server", "timestamp": "2020-05-24T20:04:13,937Z", "level": "ERROR", "component": "o.e.i.e.Engine", "cluster.name": "elasticsearch", "node.name": "els", "message": " [fincorpus][0] already closed by tragic event on the index writer", "cluster.uuid": "Ic_DMuXJTQqa4zu3wSEobw", "node.id": "12SGfI1qQkyZ-KJjF_okPA" ,
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
fatal error in thread [elasticsearch[els][refresh][T#1]], exiting
java.lang.InternalError: a fault occurred in a recent unsafe memory access operation in compiled Java code
at org.apache.lucene.index.ByteSliceReader.readByte(ByteSliceReader.java:73)
at org.apache.lucene.store.DataInput.readVInt(DataInput.java:126)
at org.apache.lucene.index.FreqProxFields$FreqProxPostingsEnum.nextPosition(FreqProxFields.java:501)
at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:150)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:937)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:347)
at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97)
at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
at org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
at org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
at org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
at org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
at org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
at org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3234)
at org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:835)
at org.elasticsearch.index.IndexService.access$200(IndexService.java:109)
at org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:967)
at org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base/java.lang.Thread.run(Thread.java:832)
{"type": "server", "timestamp": "2020-05-24T20:04:13,941Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "elasticsearch", "node.name": "els", "message": "fatal error in thread [elasticsearch[els][write][T#7]], exiting", "cluster.uuid": "Ic_DMuXJTQqa4zu3wSEobw", "node.id": "12SGfI1qQkyZ-KJjF_okPA" ,