From 5c07797a76022ab3e121a7da4372ef8a3b544eba Mon Sep 17 00:00:00 2001 From: Ben Salha Mohamed Amine <mohamed-amine.ben-salha@student-cs.fr> Date: Fri, 6 Jan 2023 09:28:53 +0100 Subject: [PATCH] metaflow discovery --- charts/jupyter-pyspark/Chart.yaml | 4 +- .../templates/configmap-metaflow.yaml | 1 + .../templates/statefulset.yaml | 12 ++++- charts/jupyter-pyspark/values.schema.json | 6 +++ charts/jupyter-pyspark/values.yaml | 5 +- charts/jupyter-python/Chart.yaml | 4 +- .../templates/configmap-metaflow.yaml | 1 + .../jupyter-python/templates/statefulset.yaml | 15 +++++- charts/jupyter-python/values.schema.json | 6 +++ charts/jupyter-python/values.yaml | 7 ++- charts/library-chart/Chart.yaml | 2 +- charts/library-chart/templates/_configmap.tpl | 49 +++++++++++++++++++ 12 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 charts/jupyter-pyspark/templates/configmap-metaflow.yaml create mode 100644 charts/jupyter-python/templates/configmap-metaflow.yaml diff --git a/charts/jupyter-pyspark/Chart.yaml b/charts/jupyter-pyspark/Chart.yaml index f8322da..8f54e8f 100644 --- a/charts/jupyter-pyspark/Chart.yaml +++ b/charts/jupyter-pyspark/Chart.yaml @@ -23,9 +23,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.13.1 +version: 1.13.2 dependencies: - name: library-chart - version: 1.3.0 + version: 1.3.1 repository: https://inseefrlab.github.io/helm-charts-interactive-services diff --git a/charts/jupyter-pyspark/templates/configmap-metaflow.yaml b/charts/jupyter-pyspark/templates/configmap-metaflow.yaml new file mode 100644 index 0000000..6d1c5e0 --- /dev/null +++ b/charts/jupyter-pyspark/templates/configmap-metaflow.yaml @@ -0,0 +1 @@ +{{ include "library-chart.configMapMetaflow" . }} \ No newline at end of file diff --git a/charts/jupyter-pyspark/templates/statefulset.yaml b/charts/jupyter-pyspark/templates/statefulset.yaml index f968655..20f633f 100644 --- a/charts/jupyter-pyspark/templates/statefulset.yaml +++ b/charts/jupyter-pyspark/templates/statefulset.yaml @@ -46,6 +46,11 @@ spec: - name: config-coresite configMap: name: {{ include "library-chart.configMapNameCoreSite" . }} + {{- if .Values.discovery.metaflow }} + - name: metaflow-config + configMap: + name: {{ include "library-chart.configMapNameMetaflow" . }} + {{- end }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -159,7 +164,12 @@ spec: - name: hive-config mountPath: /opt/hive/conf/hive-site.xml subPath: hive-site.xml - {{- end }} + {{- end }} + {{- if .Values.discovery.metaflow }} + - name: metaflow-config + mountPath: /home/{{ .Values.environment.user}}/.metaflowconfig/config.json + subPath: config.json + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/jupyter-pyspark/values.schema.json b/charts/jupyter-pyspark/values.schema.json index a8205ed..42b4c34 100644 --- a/charts/jupyter-pyspark/values.schema.json +++ b/charts/jupyter-pyspark/values.schema.json @@ -96,6 +96,12 @@ "title": "Enable mlflow discovery", "description": "discover your mlflow service", "default": true + }, + "metaflow": { + "type": "boolean", + "title": "Enable metaflow discovery", + "description": "discover your metaflow service", + "default": true } } }, diff --git a/charts/jupyter-pyspark/values.yaml b/charts/jupyter-pyspark/values.yaml index d24b498..e0c151e 100644 --- a/charts/jupyter-pyspark/values.yaml +++ b/charts/jupyter-pyspark/values.yaml @@ -99,6 +99,7 @@ repository: discovery: hive: true mlflow: true + metaflow: true hive: configMapName: "" @@ -108,7 +109,9 @@ mlflow: coresite: configMapName: "" - + +metaflow: + configMapName: "" # This is a YAML-formatted file. # Declare variables to be passed into your templates. replicaCount: 1 diff --git a/charts/jupyter-python/Chart.yaml b/charts/jupyter-python/Chart.yaml index fc5b274..58dbc15 100644 --- a/charts/jupyter-python/Chart.yaml +++ b/charts/jupyter-python/Chart.yaml @@ -22,10 +22,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.6.2 +version: 1.6.3 dependencies: - name: library-chart - version: 1.3.0 + version: 1.3.1 repository: https://inseefrlab.github.io/helm-charts-interactive-services # repository: "file://../library-chart" \ No newline at end of file diff --git a/charts/jupyter-python/templates/configmap-metaflow.yaml b/charts/jupyter-python/templates/configmap-metaflow.yaml new file mode 100644 index 0000000..6d1c5e0 --- /dev/null +++ b/charts/jupyter-python/templates/configmap-metaflow.yaml @@ -0,0 +1 @@ +{{ include "library-chart.configMapMetaflow" . }} \ No newline at end of file diff --git a/charts/jupyter-python/templates/statefulset.yaml b/charts/jupyter-python/templates/statefulset.yaml index e9b90b3..2741246 100644 --- a/charts/jupyter-python/templates/statefulset.yaml +++ b/charts/jupyter-python/templates/statefulset.yaml @@ -34,6 +34,9 @@ spec: {{- if not (empty (trim (include "library-chart.configMapHive" .)))}} checksum/hive: {{ include (print $.Template.BasePath "/configmap-hive.yaml") . | sha256sum }} {{- end }} + {{- if not (empty (trim (include "library-chart.configMapMetaflow" .)))}} + checksum/metaflow: {{ include (print $.Template.BasePath "/configmap-metaflow.yaml") . | sha256sum }} + {{- end }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -53,6 +56,11 @@ spec: configMap: name: {{ include "library-chart.configMapNameHive" . }} {{- end }} + {{- if .Values.discovery.metaflow }} + - name: metaflow-config + configMap: + name: {{ include "library-chart.configMapNameMetaflow" . }} + {{- end }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -158,7 +166,12 @@ spec: - name: hive-config mountPath: /opt/hive/conf/hive-site.xml subPath: hive-site.xml - {{- end }} + {{- end }} + {{- if .Values.discovery.metaflow }} + - name: metaflow-config + mountPath: /home/{{ .Values.environment.user}}/.metaflowconfig/config.json + subPath: config.json + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/jupyter-python/values.schema.json b/charts/jupyter-python/values.schema.json index d65323d..aac34bd 100644 --- a/charts/jupyter-python/values.schema.json +++ b/charts/jupyter-python/values.schema.json @@ -71,6 +71,12 @@ "title": "Enable mlflow discovery", "description": "discover your mlflow service", "default": true + }, + "metaflow": { + "type": "boolean", + "title": "Enable metaflow discovery", + "description": "discover your metaflow service", + "default": true } } }, diff --git a/charts/jupyter-python/values.yaml b/charts/jupyter-python/values.yaml index b218636..44627f6 100644 --- a/charts/jupyter-python/values.yaml +++ b/charts/jupyter-python/values.yaml @@ -75,7 +75,8 @@ repository: discovery: hive: true mlflow: true - + metaflow: true + hive: configMapName: "" @@ -84,7 +85,9 @@ mlflow: coresite: configMapName: "" - + +metaflow: + configMapName: "" # This is a YAML-formatted file. # Declare variables to be passed into your templates. replicaCount: 1 diff --git a/charts/library-chart/Chart.yaml b/charts/library-chart/Chart.yaml index 42f99c5..229eff5 100644 --- a/charts/library-chart/Chart.yaml +++ b/charts/library-chart/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v2 name: library-chart -version: 1.3.0 +version: 1.3.1 type: library diff --git a/charts/library-chart/templates/_configmap.tpl b/charts/library-chart/templates/_configmap.tpl index 8bb0f7d..5db7c38 100644 --- a/charts/library-chart/templates/_configmap.tpl +++ b/charts/library-chart/templates/_configmap.tpl @@ -307,3 +307,52 @@ data: {{- end }} {{- end }} {{- end }} + + +{{/* Create the name of the config map Metaflow to use */}} +{{- define "library-chart.configMapNameMetaflow" -}} +{{- $name:= (printf "%s-configmapmetaflow" (include "library-chart.fullname" .)) }} +{{- default $name .Values.metaflow.configMapName }} +{{- end }} + +{{/* ConfigMap for config.json for Metaflow */}} +{{- define "library-chart.metaflow" -}} +{{- $namespace:= .Release.Namespace -}} +{{- printf "{" }} +{{- printf "\"METAFLOW_DEFAULT_METADATA\": \"service\"," | indent 2 }} +{{- printf "\"METAFLOW_KUBERNETES_SERVICE_ACCOUNT\": \"default\"," | indent 2 }} +{{- printf "\"METAFLOW_S3_ENDPOINT_URL\": \"https://minio.lab.sspcloud.fr\"," | indent 2 }} +{{- if .Values.discovery.metaflow -}} +{{- range $index, $secret := (lookup "v1" "Secret" .Release.Namespace "").items -}} +{{- if (index $secret "metadata" "annotations") -}} +{{- if and (index $secret "metadata" "annotations" "onyxia/discovery") (eq "metaflow" (index $secret "metadata" "annotations" "onyxia/discovery" | toString)) -}} +{{- $host:= ( index $secret.data "host" | default "") | b64dec -}} +{{- $s3Bucket := (index $secret.data "s3Bucket" | default "") | b64dec -}} +{{- $s3Secret := (index $secret.data "s3Secret" | default "") | b64dec -}} +{{- printf "\"METAFLOW_KUBERNETES_NAMESPACE\": \"%s\"," $namespace | indent 2 }} +{{- printf "\"METAFLOW_SERVICE_URL\": \"%s\"," $host | indent 2 }} +{{- printf "\"METAFLOW_KUBERNETES_SECRETS\": \"%s\"," $s3Secret | indent 2 }} +{{- printf "\"METAFLOW_DATASTORE_SYSROOT_S3\": \"%s\"," $s3Bucket | indent 2 }} +{{- printf "\"METAFLOW_DATATOOLS_SYSROOT_S3\": \"%s\"," $s3Bucket | indent 2 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- printf "\"METAFLOW_DEFAULT_DATASTORE\": \"s3\"" | indent 2 }} +{{- printf "}" }} +{{- end }} + + +{{/* Template to generate a ConfigMap for Metaflow */}} +{{- define "library-chart.configMapMetaflow" -}} +{{- $context:= . -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "library-chart.configMapNameMetaflow" $context }} + labels: + {{- include "library-chart.labels" $context | nindent 4 }} +data: + config.json: | + {{- include "library-chart.metaflow" . | nindent 4}} +{{- end }} \ No newline at end of file -- GitLab