Intermittent socket connection errors when dealing with document automation data

I have a process which sends files through document automation for data extraction.

After extraction, I am doing some post-processing on the JSON data using the “Document Extraction: Get document data” command and the “Document Extraction: Update document data” command.

Most of the time, this works fine. However, 10-20 percent of documents each day fail at this step due to socket connection errors.

19 failures out of 111 documents for the most recent batch.

What could be causing this?

My basic error logs report this when failing to get the data:

java.net.SocketException: Connection reset by peer: socket write error 

and this when failing to update the data:

java.util.concurrent.ExecutionException: net.jodah.failsafe.FailsafeException: java.net.SocketException: Connection reset by peer: socket write error 

The more detailed log file shows:

 I/O exception (java.net.SocketException) caught when processing request to {s}->https://aapd-prod-use1-ee-us-east-1.s3.amazonaws.com:443: Connection reset

execute(GetDocumentDataCommandCommand.java:60) - java.net.SocketException: Connection reset by peer: socket write error net.jodah.failsafe.FailsafeException: java.net.SocketException: Connection reset by peer: socket write error
	at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:386) ~[bot-launcher.jar:?]
	at net.jodah.failsafe.FailsafeExecutor.get(FailsafeExecutor.java:68) ~[bot-launcher.jar:?]
	at com.automationanywhere.iqbot.infrastructure.restclient.RestClientV2.doHttpRequest(RestClientV2.java:245) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at com.automationanywhere.iqbot.infrastructure.restclient.RestClientV2.doGet(RestClientV2.java:229) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at com.automationanywhere.iqbot.botcommand.commands.common.DACommandsRestClient.getResponse(DACommandsRestClient.java:60) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at com.automationanywhere.iqbot.botcommand.commands.document.application.DocumentDataService.downloadFile(DocumentDataService.java:339) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at com.automationanywhere.iqbot.botcommand.commands.document.application.DocumentDataService.validateAndDownload(DocumentDataService.java:307) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at com.automationanywhere.iqbot.botcommand.commands.document.application.DocumentDataService.downloadDocument(DocumentDataService.java:96) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at com.automationanywhere.iqbot.botcommand.commands.document.GetDocumentDataCommand.getIqbotValidatorData(GetDocumentDataCommand.java:211) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at com.automationanywhere.iqbot.botcommand.commands.document.GetDocumentDataCommand.compute(GetDocumentDataCommand.java:171) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at com.automationanywhere.iqbot.botcommand.commands.document.GetDocumentDataCommandCommand.execute(GetDocumentDataCommandCommand.java:49) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at com.automationanywhere.toolchain.runtime.BotCommand.execute(BotCommand.java:73) ~[bot-runtime.jar:?]
	at com.automationanywhere.toolchain.runtime.BotCommand.execute(BotCommand.java:95) ~[bot-runtime.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
	at java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
	at com.automationanywhere.bot.metrics.TimerInvocationHandler.invoke(TimerInvocationHandler.java:66) ~[bot-launcher.jar:?]
	at com.sun.proxy.$Proxy30.execute(Unknown Source) ~[?:?]
	at com.automationanywhere.botrunner.bot.Bot_PO_Validation_Bot.executeNode9(Bot_PO_Validation_Bot.java:415) ~[?:?]
	at com.automationanywhere.botrunner.bot.Bot_PO_Validation_Bot.play(Bot_PO_Validation_Bot.java:1145) ~[?:?]
	at com.automationanywhere.botrunner.Main.play(Main.java:63) ~[?:?]
	at com.automationanywhere.botengine.service.impl.BotLauncherImpl.runBot(BotLauncherImpl.java:22) ~[bot-launcher.jar:?]
	at com.automationanywhere.botengine.service.impl.DispatcherImpl.lambda$start$9(DispatcherImpl.java:641) ~[bot-launcher.jar:?]
	at com.automationanywhere.botengine.utils.ThreadUtil.lambda$withThreadContext$0(ThreadUtil.java:20) ~[bot-launcher.jar:?]
	at java.util.concurrent.CompletableFuture$AsyncSupply.run(Unknown Source) [?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]
	at java.lang.Thread.run(Unknown Source) [?:?]
Caused by: java.net.SocketException: Connection reset by peer: socket write error
	at java.net.SocketOutputStream.socketWrite0(Native Method) ~[?:?]
	at java.net.SocketOutputStream.socketWrite(Unknown Source) ~[?:?]
	at java.net.SocketOutputStream.write(Unknown Source) ~[?:?]
	at sun.security.ssl.SSLSocketOutputRecord.encodeChangeCipherSpec(Unknown Source) ~[?:?]
	at sun.security.ssl.OutputRecord.changeWriteCiphers(Unknown Source) ~[?:?]
	at sun.security.ssl.ServerHello$T13ServerHelloConsumer.consume(Unknown Source) ~[?:?]
	at sun.security.ssl.ServerHello$ServerHelloConsumer.onServerHello(Unknown Source) ~[?:?]
	at sun.security.ssl.ServerHello$ServerHelloConsumer.consume(Unknown Source) ~[?:?]
	at sun.security.ssl.SSLHandshake.consume(Unknown Source) ~[?:?]
	at sun.security.ssl.HandshakeContext.dispatch(Unknown Source) ~[?:?]
	at sun.security.ssl.HandshakeContext.dispatch(Unknown Source) ~[?:?]
	at sun.security.ssl.TransportContext.dispatch(Unknown Source) ~[?:?]
	at sun.security.ssl.SSLTransport.decode(Unknown Source) ~[?:?]
	at sun.security.ssl.SSLSocketImpl.decode(Unknown Source) ~[?:?]
	at sun.security.ssl.SSLSocketImpl.readHandshakeRecord(Unknown Source) ~[?:?]
	at sun.security.ssl.SSLSocketImpl.startHandshake(Unknown Source) ~[?:?]
	at sun.security.ssl.SSLSocketImpl.startHandshake(Unknown Source) ~[?:?]
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:436) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:384) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:376) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:108) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at com.automationanywhere.iqbot.infrastructure.restclient.RestClientV2.lambda$doHttpRequest$0(RestClientV2.java:245) ~[bot-command-iqbot-extraction360-3.38.12.jar:?]
	at net.jodah.failsafe.Functions.lambda$get$0(Functions.java:48) ~[bot-launcher.jar:?]
	at net.jodah.failsafe.RetryPolicyExecutor.lambda$supply$0(RetryPolicyExecutor.java:66) ~[bot-launcher.jar:?]
	at net.jodah.failsafe.Execution.executeSync(Execution.java:128) ~[bot-launcher.jar:?]
	at net.jodah.failsafe.FailsafeExecutor.call(FailsafeExecutor.java:379) ~[bot-launcher.jar:?]
	... 28 more