We've been indexing data on a 45 node cluster with 100 shards and 3
replicas, but our indexing processes have been stopping due to errors.  
On the server side the error is "Error logging add". Stack trace:

2017-07-17 12:29:24.057 INFO  (qtp985934102-5161548) [c:UNCLASS
s:shard58 r:core_node290 x:UNCLASS_shard58_replica1]
o.a.s.u.p.LogUpdateProcessorFactory [UNCLASS_shard58_replica1]
webapp=/solr path=/update
params={update.distrib=FROMLEADER&update.chain=add-unknown-fields-to-the-schema&distrib.from=http://tarvos:9100/solr/UNCLASS_shard58_replica2/&wt=javabin&version=2}{add=[COLLECT20003218348784
(1573172872544780288), COLLECT20003218351447 (1573172872620277760),
COLLECT20003218353085 (1573172872625520640), COLLECT20003218357937
(1573172872627617792), COLLECT20003218361860 (1573172872629714944),
COLLECT20003218362535 (1573172872631812096)]} 0 171
2017-07-17 12:29:24.160 INFO  (qtp985934102-5160762) [c:UNCLASS
s:shard13 r:core_node81 x:UNCLASS_shard13_replica1]
o.a.s.u.p.LogUpdateProcessorFactory [UNCLASS_shard13_replica1]
webapp=/solr path=/update
params={update.distrib=FROMLEADER&update.chain=add-unknown-fields-to-the-schema&distrib.from=http://tarvos:9100/solr/UNCLASS_shard13_replica2/&wt=javabin&version=2}{add=[COLLECT20003218344436
(1573172872538488832), COLLECT20003218347497 (1573172872620277760),
COLLECT20003218351645 (1573172872625520640), COLLECT20003218356965
(1573172872629714944), COLLECT20003218357775 (1573172872632860672),
COLLECT20003218358017 (1573172872646492160), COLLECT20003218358152
(1573172872650686464), COLLECT20003218359395 (1573172872651735040),
COLLECT20003218362571 (1573172872652783616)]} 0 274
2017-07-17 12:29:24.163 INFO  (qtp985934102-5161057) [c:UNCLASS
s:shard43 r:core_node108 x:UNCLASS_shard43_replica1]
o.a.s.u.p.LogUpdateProcessorFactory [UNCLASS_shard43_replica1]
webapp=/solr path=/update
params={update.distrib=FROMLEADER&update.chain=add-unknown-fields-to-the-schema&distrib.from=http://tarvos:9100/solr/UNCLASS_shard43_replica2/&wt=javabin&version=2}{}
0 0
2017-07-17 12:29:24.164 ERROR (qtp985934102-5161057) [c:UNCLASS
s:shard43 r:core_node108 x:UNCLASS_shard43_replica1]
o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error
logging add
         at
org.apache.solr.update.TransactionLog.write(TransactionLog.java:418)
         at org.apache.solr.update.UpdateLog.add(UpdateLog.java:532)
         at org.apache.solr.update.UpdateLog.add(UpdateLog.java:516)
         at
org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1113)
         at
org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:748)
         at
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)
         at
org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:98)
         at
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:180)
         at
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:136)
         at
org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:306)
         at
org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:251)
         at
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:122)
         at
org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:271)
         at
org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:251)
         at
org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:173)
         at
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:187)
         at
org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:108)
         at
org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:55)
         at
org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
         at
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
         at
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2477)
         at
org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:723)
         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:529)
         at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
         at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
         at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
         at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
         at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
         at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
         at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
         at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
         at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
         at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
         at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
         at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
         at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
         at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
         at
org.eclipse.jetty.server.handler.HandlerWrapper
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB