@<1734020208089108480:profile|WickedHare16> - please try configuring the cookieDomain

clearml:
  cookieDomain: ""

You should set it as your base domain, example pixis.internal , without any api or files in front of it

Hi @<1734020208089108480:profile|WickedHare16> , what is the image of the apiserver that you are running?

Please take a look here:
None
Does it match you scenario? Can you try the suggested workaround?

One more thing, I am uploading the plot, but it is not visible

It is tag: "1.16.1-499" , and setting cookieDomain in clearml values.yaml, fixed it

I am not using external elastic search, that why not changed this value

@<1722061389024989184:profile|ResponsiveKoala38>

hi @<1729671499981262848:profile|CooperativeKitten94> , how can I fix the above issues?

Got it, and are you using external Mongo or Redis?

thanks @<1722061389024989184:profile|ResponsiveKoala38>

Thanks @<1729671499981262848:profile|CooperativeKitten94> it working now.

Hey @<1734020208089108480:profile|WickedHare16> - Not 100% sure this is the issue, but I noticed a wrong configuration in your values.
You configured both these:

elasticsearch:
  enabled: true


externalServices:
  # -- Existing ElasticSearch connectionstring if elasticsearch.enabled is false (example in values.yaml)
  elasticsearchConnectionString: "[{\"host\":\"es_hostname1\",\"port\":9200},{\"host\":\"es_hostname2\",\"port\":9200},{\"host\":\"es_hostname3\",\"port\":9200}]"

Please, if you want to use your own Elasticsearch, you will have to turn elasticsearch.enabled=false and just define externalServices.elasticsearchConnectionString . Setting both will fallback to the built-in elasticsearch and eventually cause issues.
Try changing this one and try again to see if it resolves 🙂

Mongo

clearml:
  cookieName: "clearml-token-k8s"
  cookieDomain: ""
  # -- Company name
  defaultCompany: "pixis"
  # -- Api Server basic auth key
  apiserverKey: GGS9F4M6XB2DXJ5AFT9F
  # -- Api Server basic auth secret
  apiserverSecret: 2oGujVFhPfaozhpuz2GzQfA5OyxmMsR3WVJpsCR5hrgHFs20PO
  # -- File Server basic auth key
  fileserverKey: XXCRJ123CEE2KSQ068WO
  # -- File Server basic auth secret
  fileserverSecret: YIy8EVAC7QCT4FtgitxAQGyW7xRHDZ4jpYlTE7HKiscpORl1hG
  # -- Readiness probe basic auth key
  readinessprobeKey: GK4PRTVT3706T25K6BA1
  # -- Readiness probe basic auth secret
  readinessprobeSecret: ymLh1ok5k5xNUQfS944Xdx9xjf0wueokqKM2dMZfHuH9ayItG2
  # -- Secure Auth secret
  secureAuthTokenSecret: ymLh1ok5k5xNUQfS944Xdx9xjf0wueokqKM2dMZfHuH9ayItG2
  # -- Test Server basic auth key
  testUserKey: "ENP39EQM4SLACGD5FXB7"
  # -- Test File Server basic auth secret
  testUserSecret: "lPcm0imbcBZ8mwgO7tpadutiS3gnJD05x9j7afwXPS35IKbpiQ"
  # -- Override the API Urls displayed when showing an example of the SDK's clearml.conf configuration
  clientConfigurationApiUrl: "

"
  # -- Override the Files Urls displayed when showing an example of the SDK's clearml.conf configuration
  clientConfigurationFilesUrl: "

"
  # -- Pass Clearml secrets using an existing secret
  # must contain the keys: apiserver_key, apiserver_secret, secure_auth_token_secret, test_user_key, test_user_secret
  existingSecret: ""

apiserver:
  enabled: true
  prepopulateEnabled: true
  serviceAccountName: clearml
  service:
    type: ClusterIP
  extraEnvs:
    - name: CLEARML_MONGODB_SERVICE_CONNECTION_STRING
      value: monogdburl
  ingress:
    enabled: true
    ingressClassName: alb
    hostName: clearml-api-dev.pixis.internal
    tlsSecretName: ""
    annotations:
      alb.ingress.kubernetes.io/backend-protocol: HTTP
      alb.ingress.kubernetes.io/certificate-arn: certificateurm
      alb.ingress.kubernetes.io/group.name: flyte
      alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80,"HTTPS": 443}]'
      alb.ingress.kubernetes.io/scheme: internal
      alb.ingress.kubernetes.io/ssl-passthrough: "true"
      alb.ingress.kubernetes.io/ssl-ports: https
      alb.ingress.kubernetes.io/target-type: ip
    path: "/"
  additionalConfigs:
    apiserver.conf: |
      auth {
        fixed_users {
          enabled: true
          pass_hashed: true
          users: [
            {
              username: "pai"
              password: "hassedpassword"
              name: "Pai team"
            },
            {
              username: "tcai"
              password: "hassedpassword"
              name: "TCai team"
            },
          ]
        }
      }  

fileserver:
  enabled: true
  serviceAccountName: clearml
  service:
    type: ClusterIP
  replicaCount: 1
  storage:
    enabled: true
  extraEnvs:
    - name: CLEARML_MONGODB_SERVICE_CONNECTION_STRING
      value: mongodburl
  ingress:
    enabled: true
    ingressClassName: alb
    hostName: clearml-files-dev.pixis.internal
    tlsSecretName: ""
    annotations:
      alb.ingress.kubernetes.io/backend-protocol: HTTP
      alb.ingress.kubernetes.io/certificate-arn: certificatearn
      alb.ingress.kubernetes.io/group.name: flyte
      alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80,"HTTPS": 443}]'
      alb.ingress.kubernetes.io/scheme: internal
      alb.ingress.kubernetes.io/ssl-passthrough: "true"
      alb.ingress.kubernetes.io/ssl-ports: https
      alb.ingress.kubernetes.io/target-type: ip
    path: "/"
  

webserver:
  enabled: true
  serviceAccountName: clearml
  service:
    type: ClusterIP
  extraEnvs:
    - name: CLEARML_MONGODB_SERVICE_CONNECTION_STRING
      value: mongodb url
  ingress:
    enabled: true
    ingressClassName: alb
    hostName: clearml-app-dev.pixis.internal
    tlsSecretName: ""
    annotations:
      alb.ingress.kubernetes.io/backend-protocol: HTTP
      alb.ingress.kubernetes.io/certificate-arn: certificate arn
      alb.ingress.kubernetes.io/group.name: flyte
      alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80,"HTTPS": 443}]'
      alb.ingress.kubernetes.io/scheme: internal
      alb.ingress.kubernetes.io/ssl-passthrough: "true"
      alb.ingress.kubernetes.io/ssl-ports: https
      alb.ingress.kubernetes.io/target-type: ip
    path: "/"
  

redis:
  enabled: true

mongodb:
  enabled: false

elasticsearch:
  enabled: true


externalServices:
  # -- Existing ElasticSearch connectionstring if elasticsearch.enabled is false (example in values.yaml)
  elasticsearchConnectionString: "[{\"host\":\"es_hostname1\",\"port\":9200},{\"host\":\"es_hostname2\",\"port\":9200},{\"host\":\"es_hostname3\",\"port\":9200}]"
  # -- Existing MongoDB connection string for BACKEND to use if mongodb.enabled is false (example in values.yaml)
  mongodbConnectionStringAuth: ""
  # -- Existing MongoDB connection string for AUTH to use if mongodb.enabled is false (example in values.yaml)
  mongodbConnectionStringBackend: ""

Hey @<1734020208089108480:profile|WickedHare16> , could you please share your override values file for the clearml helm chart?

🙂

@<1722061389024989184:profile|ResponsiveKoala38>

@<1734020208089108480:profile|WickedHare16> Can you please share an example plot url that is not opened in UI but you can see it in a separate tab?

plots are visible if I copy the url and open in new tab

@<1734020208089108480:profile|WickedHare16> Do you mean that you see the plots now? Are there still any _attempt_serialize_numpy errors in the apiserver logs?

not the plots, but the notebook preview

this is exactly same isssue

yes

do we have any config for this

This is continuous error in apiserver

[2024-09-06 08:46:59,887] [9] [ERROR] [clearml.queue_metrics] Failed collecting queue metrics: `np.float_` was removed in the NumPy 2.0 release. Use `np.float64` instead.
Traceback (most recent call last):
File "/opt/clearml/apiserver/bll/queue/queue_metrics.py", line 311, in start
queue_metrics.log_queue_metrics_to_es(queue.company, [queue])
File "/opt/clearml/apiserver/bll/queue/queue_metrics.py", line 83, in log_queue_metrics_to_es
self.es.index(index=es_index, document=queue_doc)
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/utils.py", line 426, in wrapped
return api(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/__init__.py", line 2280, in index
return self.perform_request( # type: ignore[return-value]
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 285, in perform_request
meta, resp_body = self.transport.perform_request(
File "/usr/local/lib/python3.9/site-packages/elastic_transport/_transport.py", line 319, in perform_request
request_body = self.serializers.dumps(
File "/usr/local/lib/python3.9/site-packages/elastic_transport/_serializer.py", line 224, in dumps
return self.get_serializer(mimetype).dumps(data)
File "/usr/local/lib/python3.9/site-packages/elastic_transport/_serializer.py", line 120, in dumps
return self.json_dumps(data)
File "/usr/local/lib/python3.9/site-packages/elastic_transport/_serializer.py", line 90, in json_dumps
return json.dumps(
File "/usr/local/lib/python3.9/json/__init__.py", line 234, in dumps
return cls(
File "/usr/local/lib/python3.9/json/encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/usr/local/lib/python3.9/json/encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "/usr/local/lib/python3.9/site-packages/elasticsearch/serializer.py", line 69, in default
serialized, value = _attempt_serialize_numpy_or_pandas(data)
File "/usr/local/lib/python3.9/site-packages/elasticsearch/serializer.py", line 125, in _attempt_serialize_numpy_or_pandas
serialized, value = _attempt_serialize_numpy(data)
File "/usr/local/lib/python3.9/site-packages/elasticsearch/serializer.py", line 160, in _attempt_serialize_numpy
np.float_,
File "/usr/local/lib/python3.9/site-packages/numpy/__init__.py", line 397, in __getattr__
raise AttributeError(
AttributeError: `np.float_` was removed in the NumPy 2.0 release. Use `np.float64` instead.