Failing to push large docker image > 10 GB to Artifactory on OpenShift – Nginx

We are using Artifactory on Openshift as docker registry. the Installation happened via helm chart from jFrog. So far everything is working but uploading large docker images > 10GB to the registry.

We are using the Nginx reverse proxy in one pod and the artifactory in an other pod. It should be behaving like Nginx is not on the same server as artifactory itself.

On console it looks like the push st working. The smaller layers are pushed and the large one is also uploading. After some seconds, it starts re uploading again.

Artifactory throws this error

2021-08-23T05:41:53.624Z [jfrt ] [ERROR] [                ] [.j.a.c.g.GrpcStreamObserver:97] [default-executor-755] - refreshing affected platform config stream - got an error (status: Status{code=INTERNAL, description=Received unexpected EOS on DATA frame from server., cause=null})
    io.grpc.StatusRuntimeException: INTERNAL: Received unexpected EOS on DATA frame from server.
        at io.grpc.Status.asRuntimeException(Status.java:533)
        at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:478)
        at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
        at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
        at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
        at org.jfrog.access.client.grpc.AuthorizationInterceptor$AuthenticatedClientCall$RejoiningClientCallListener.onClose(AuthorizationInterceptor.java:73)
        at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:413)
        at io.grpc.internal.ClientCallImpl.access$500(ClientCallImpl.java:66)
        at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:742)
        at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:721)
        at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
        at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base/java.lang.Thread.run(Thread.java:834)

the Artifactory Nginx conf looks like this (mostly generated by Artifactory):

 server {
  listen 443 ssl;
  listen 80;
  server_name ~(?<repo>.+)\.my.url.ch my.url my.nonssl.url;
  
  ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
  ssl_certificate     /var/opt/jfrog/nginx/ssl/tls.crt;  
  ssl_certificate_key /var/opt/jfrog/nginx/ssl/tls.key;
  ssl_password_file   /var/opt/jfrog/nginx/ssl/tls.pass;
  ssl_ciphers         HIGH:!aNULL:!MD5;
  
  if ($http_x_forwarded_proto = '') {
    set $http_x_forwarded_proto  $scheme;
  }

  ## Application specific logs 
  rewrite ^/$ /ui/ redirect;
  rewrite ^/ui$ /ui/ redirect;
  rewrite ^/(v1|v2)/(.*) /artifactory/api/docker/$repo/$1/$2;
  chunked_transfer_encoding on;
  client_max_body_size 0;
  location / {
    proxy_read_timeout  2400s;
    proxy_pass_header   Server;
    proxy_cookie_path   ~*^/.* /;
    proxy_buffer_size 128k;
    proxy_buffers 40 128k;
    proxy_busy_buffers_size 128k;
    #proxy_buffering off; 
    #proxy_request_buffering off;
    proxy_pass          http://devlab-artifactory:8082;
    proxy_set_header    X-JFrog-Override-Base-Url $http_x_forwarded_proto://$host;
    proxy_set_header    X-Forwarded-Port  $server_port;
    proxy_set_header    X-Forwarded-Proto $http_x_forwarded_proto;
    proxy_set_header    Host              $http_host;
    proxy_set_header    X-Forwarded-For   $proxy_add_x_forwarded_for;
    add_header Strict-Transport-Security always;
    
    location ~ ^/artifactory/ {
      proxy_pass    http://artifactory:8081;
    }
  }
}

nginx.conf

# Main Nginx configuration file
worker_processes  4;


error_log  stderr warn;
pid        /tmp/nginx.pid;

events {
  worker_connections  1024;
}


http {
  include       /etc/nginx/mime.types;
  default_type  application/octet-stream;

  variables_hash_max_size 1024;
  variables_hash_bucket_size 64;
  server_names_hash_max_size 4096;
  server_names_hash_bucket_size 128;
  types_hash_max_size 2048;
  types_hash_bucket_size 64;
  proxy_read_timeout 2400s;
  client_header_timeout 2400s;
  client_body_timeout 2400s;
  proxy_connect_timeout 75s;
  proxy_send_timeout 2400s;
  proxy_buffer_size 128k;
  proxy_buffers 40 128k;
  proxy_busy_buffers_size 128k;
  proxy_temp_file_write_size 250m;
  proxy_http_version 1.1;
  client_max_body_size 100G;
  client_body_buffer_size 128k;
  client_body_in_file_only clean;

  log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
  '$status $body_bytes_sent "$http_referer" '
  '"$http_user_agent" "$http_x_forwarded_for"';

  log_format timing 'ip = $remote_addr '
  'user = "$remote_user" '
  'local_time = "$time_local" '
  'host = $host '
  'request = "$request" '
  'status = $status '
  'bytes = $body_bytes_sent '
  'upstream = "$upstream_addr" '
  'upstream_time = $upstream_response_time '
  'request_time = $request_time '
  'referer = "$http_referer" '
  'UA = "$http_user_agent"';

  access_log  /var/opt/jfrog/nginx/logs/access.log  timing;

  sendfile        on;
  #tcp_nopush     on;

  keepalive_timeout  65;

  #gzip  on;

  include /etc/nginx/conf.d/*.conf;

}

We tried a lot of things here. Larger client body sizes or disabling proxy buffering but I did not manage to upload more than 5.6 GB

On Harbor I managed to upload this kind of image, so it should be somehow possible to do the same thing on Artifactory.