From 5c07797a76022ab3e121a7da4372ef8a3b544eba Mon Sep 17 00:00:00 2001
From: Ben Salha Mohamed Amine <mohamed-amine.ben-salha@student-cs.fr>
Date: Fri, 6 Jan 2023 09:28:53 +0100
Subject: [PATCH] metaflow discovery

---
 charts/jupyter-pyspark/Chart.yaml             |  4 +-
 .../templates/configmap-metaflow.yaml         |  1 +
 .../templates/statefulset.yaml                | 12 ++++-
 charts/jupyter-pyspark/values.schema.json     |  6 +++
 charts/jupyter-pyspark/values.yaml            |  5 +-
 charts/jupyter-python/Chart.yaml              |  4 +-
 .../templates/configmap-metaflow.yaml         |  1 +
 .../jupyter-python/templates/statefulset.yaml | 15 +++++-
 charts/jupyter-python/values.schema.json      |  6 +++
 charts/jupyter-python/values.yaml             |  7 ++-
 charts/library-chart/Chart.yaml               |  2 +-
 charts/library-chart/templates/_configmap.tpl | 49 +++++++++++++++++++
 12 files changed, 102 insertions(+), 10 deletions(-)
 create mode 100644 charts/jupyter-pyspark/templates/configmap-metaflow.yaml
 create mode 100644 charts/jupyter-python/templates/configmap-metaflow.yaml

diff --git a/charts/jupyter-pyspark/Chart.yaml b/charts/jupyter-pyspark/Chart.yaml
index f8322da..8f54e8f 100644
--- a/charts/jupyter-pyspark/Chart.yaml
+++ b/charts/jupyter-pyspark/Chart.yaml
@@ -23,9 +23,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.13.1
+version: 1.13.2
 
 dependencies:
   - name: library-chart
-    version: 1.3.0
+    version: 1.3.1
     repository: https://inseefrlab.github.io/helm-charts-interactive-services
diff --git a/charts/jupyter-pyspark/templates/configmap-metaflow.yaml b/charts/jupyter-pyspark/templates/configmap-metaflow.yaml
new file mode 100644
index 0000000..6d1c5e0
--- /dev/null
+++ b/charts/jupyter-pyspark/templates/configmap-metaflow.yaml
@@ -0,0 +1 @@
+{{ include "library-chart.configMapMetaflow" . }}
\ No newline at end of file
diff --git a/charts/jupyter-pyspark/templates/statefulset.yaml b/charts/jupyter-pyspark/templates/statefulset.yaml
index f968655..20f633f 100644
--- a/charts/jupyter-pyspark/templates/statefulset.yaml
+++ b/charts/jupyter-pyspark/templates/statefulset.yaml
@@ -46,6 +46,11 @@ spec:
         - name: config-coresite
           configMap:
             name: {{ include "library-chart.configMapNameCoreSite" . }}
+        {{- if .Values.discovery.metaflow }}
+        - name: metaflow-config
+          configMap:
+            name: {{ include "library-chart.configMapNameMetaflow" . }}
+        {{- end }}
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
@@ -159,7 +164,12 @@ spec:
             - name: hive-config
               mountPath: /opt/hive/conf/hive-site.xml
               subPath: hive-site.xml
-            {{- end }}         
+            {{- end }}  
+            {{- if .Values.discovery.metaflow }}
+            - name: metaflow-config
+              mountPath: /home/{{ .Values.environment.user}}/.metaflowconfig/config.json
+              subPath: config.json
+            {{- end }}       
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
diff --git a/charts/jupyter-pyspark/values.schema.json b/charts/jupyter-pyspark/values.schema.json
index a8205ed..42b4c34 100644
--- a/charts/jupyter-pyspark/values.schema.json
+++ b/charts/jupyter-pyspark/values.schema.json
@@ -96,6 +96,12 @@
           "title": "Enable mlflow discovery",
           "description": "discover your mlflow service",
           "default": true
+        },
+        "metaflow": {
+          "type": "boolean",
+          "title": "Enable metaflow discovery",
+          "description": "discover your metaflow service",
+          "default": true
         }
       }
     },
diff --git a/charts/jupyter-pyspark/values.yaml b/charts/jupyter-pyspark/values.yaml
index d24b498..e0c151e 100644
--- a/charts/jupyter-pyspark/values.yaml
+++ b/charts/jupyter-pyspark/values.yaml
@@ -99,6 +99,7 @@ repository:
 discovery:
   hive: true
   mlflow: true
+  metaflow: true
 
 hive:
   configMapName: ""
@@ -108,7 +109,9 @@ mlflow:
 
 coresite:
   configMapName: ""
-  
+
+metaflow: 
+  configMapName: ""  
   # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 replicaCount: 1
diff --git a/charts/jupyter-python/Chart.yaml b/charts/jupyter-python/Chart.yaml
index fc5b274..58dbc15 100644
--- a/charts/jupyter-python/Chart.yaml
+++ b/charts/jupyter-python/Chart.yaml
@@ -22,10 +22,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.6.2
+version: 1.6.3
 
 dependencies:
   - name: library-chart
-    version: 1.3.0
+    version: 1.3.1
     repository: https://inseefrlab.github.io/helm-charts-interactive-services
    # repository: "file://../library-chart"
\ No newline at end of file
diff --git a/charts/jupyter-python/templates/configmap-metaflow.yaml b/charts/jupyter-python/templates/configmap-metaflow.yaml
new file mode 100644
index 0000000..6d1c5e0
--- /dev/null
+++ b/charts/jupyter-python/templates/configmap-metaflow.yaml
@@ -0,0 +1 @@
+{{ include "library-chart.configMapMetaflow" . }}
\ No newline at end of file
diff --git a/charts/jupyter-python/templates/statefulset.yaml b/charts/jupyter-python/templates/statefulset.yaml
index e9b90b3..2741246 100644
--- a/charts/jupyter-python/templates/statefulset.yaml
+++ b/charts/jupyter-python/templates/statefulset.yaml
@@ -34,6 +34,9 @@ spec:
         {{- if not (empty (trim (include "library-chart.configMapHive" .)))}}
         checksum/hive: {{ include (print $.Template.BasePath "/configmap-hive.yaml") . | sha256sum }}
         {{- end }}
+        {{- if not (empty (trim (include "library-chart.configMapMetaflow" .)))}}
+        checksum/metaflow: {{ include (print $.Template.BasePath "/configmap-metaflow.yaml") . | sha256sum }}
+        {{- end }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -53,6 +56,11 @@ spec:
           configMap:
             name: {{ include "library-chart.configMapNameHive" . }}
         {{- end }}
+        {{- if .Values.discovery.metaflow }}
+        - name: metaflow-config
+          configMap:
+            name: {{ include "library-chart.configMapNameMetaflow" . }}
+        {{- end }}
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
@@ -158,7 +166,12 @@ spec:
             - name: hive-config
               mountPath: /opt/hive/conf/hive-site.xml
               subPath: hive-site.xml
-            {{- end }}      
+            {{- end }}  
+            {{- if .Values.discovery.metaflow }}
+            - name: metaflow-config
+              mountPath: /home/{{ .Values.environment.user}}/.metaflowconfig/config.json
+              subPath: config.json
+            {{- end }}     
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
diff --git a/charts/jupyter-python/values.schema.json b/charts/jupyter-python/values.schema.json
index d65323d..aac34bd 100644
--- a/charts/jupyter-python/values.schema.json
+++ b/charts/jupyter-python/values.schema.json
@@ -71,6 +71,12 @@
                 "title": "Enable mlflow discovery",
                 "description": "discover your mlflow service",
                 "default": true
+              },
+              "metaflow": {
+                "type": "boolean",
+                "title": "Enable metaflow discovery",
+                "description": "discover your metaflow service",
+                "default": true
               }
             }
           },
diff --git a/charts/jupyter-python/values.yaml b/charts/jupyter-python/values.yaml
index b218636..44627f6 100644
--- a/charts/jupyter-python/values.yaml
+++ b/charts/jupyter-python/values.yaml
@@ -75,7 +75,8 @@ repository:
 discovery:
   hive: true
   mlflow: true
-
+  metaflow: true
+  
 hive:
   configMapName: ""
 
@@ -84,7 +85,9 @@ mlflow:
 
 coresite:
   configMapName: ""
-  
+
+metaflow: 
+  configMapName: ""
   # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 replicaCount: 1
diff --git a/charts/library-chart/Chart.yaml b/charts/library-chart/Chart.yaml
index 42f99c5..229eff5 100644
--- a/charts/library-chart/Chart.yaml
+++ b/charts/library-chart/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: library-chart
-version: 1.3.0
+version: 1.3.1
 type: library
diff --git a/charts/library-chart/templates/_configmap.tpl b/charts/library-chart/templates/_configmap.tpl
index 8bb0f7d..5db7c38 100644
--- a/charts/library-chart/templates/_configmap.tpl
+++ b/charts/library-chart/templates/_configmap.tpl
@@ -307,3 +307,52 @@ data:
   {{- end }}
 {{- end }}
 {{- end }}
+
+
+{{/* Create the name of the config map Metaflow to use */}}
+{{- define "library-chart.configMapNameMetaflow" -}}
+{{- $name:= (printf "%s-configmapmetaflow" (include "library-chart.fullname" .)) }}
+{{- default $name .Values.metaflow.configMapName }}
+{{- end }}
+
+{{/* ConfigMap for config.json for Metaflow */}}
+{{- define "library-chart.metaflow" -}}
+{{- $namespace:= .Release.Namespace -}}
+{{- printf "{" }}
+{{- printf "\"METAFLOW_DEFAULT_METADATA\": \"service\"," | indent 2 }}
+{{- printf "\"METAFLOW_KUBERNETES_SERVICE_ACCOUNT\": \"default\"," | indent 2 }}
+{{- printf "\"METAFLOW_S3_ENDPOINT_URL\": \"https://minio.lab.sspcloud.fr\"," | indent 2 }}
+{{- if .Values.discovery.metaflow -}}
+{{- range $index, $secret := (lookup "v1" "Secret" .Release.Namespace "").items -}}
+{{- if (index $secret "metadata" "annotations") -}}
+{{- if and (index $secret "metadata" "annotations" "onyxia/discovery") (eq "metaflow" (index $secret "metadata" "annotations" "onyxia/discovery" | toString)) -}}
+{{- $host:= ( index $secret.data "host" | default "") | b64dec  -}}
+{{- $s3Bucket := (index $secret.data "s3Bucket" | default "") | b64dec -}}
+{{- $s3Secret := (index $secret.data "s3Secret" | default "") | b64dec -}}
+{{- printf "\"METAFLOW_KUBERNETES_NAMESPACE\": \"%s\"," $namespace | indent 2 }}
+{{- printf "\"METAFLOW_SERVICE_URL\": \"%s\"," $host | indent 2 }}
+{{- printf "\"METAFLOW_KUBERNETES_SECRETS\": \"%s\"," $s3Secret | indent 2 }}
+{{- printf "\"METAFLOW_DATASTORE_SYSROOT_S3\": \"%s\"," $s3Bucket | indent 2 }}
+{{- printf "\"METAFLOW_DATATOOLS_SYSROOT_S3\": \"%s\"," $s3Bucket | indent 2 }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- printf "\"METAFLOW_DEFAULT_DATASTORE\": \"s3\"" | indent 2 }}
+{{- printf "}" }}
+{{- end }}
+
+
+{{/* Template to generate a ConfigMap for Metaflow */}}
+{{- define "library-chart.configMapMetaflow" -}}
+{{- $context:= . -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "library-chart.configMapNameMetaflow" $context }}
+  labels:
+    {{- include "library-chart.labels" $context | nindent 4 }}
+data:
+  config.json: |
+  {{- include "library-chart.metaflow" . | nindent 4}}
+{{- end }}
\ No newline at end of file
-- 
GitLab