[GPII-3188]: First crack at smoke testing for GCP infra with Locust

.gitignore

@@ -10,6 +10,7 @@ gcp/live/*/compose.env
 gcp/live/*/*secrets/**
 gcp/live/*/*values/**
 common/live/*/*secrets/**
+common/charts/locust/tasks/*
 aws/modules/deploy/secrets/*-secrets.yml
 aws/prereqs/dev/terraform.tfstate*
 tags

.gitlab-ci.yml

@@ -77,6 +77,8 @@ gcp-dev:
     - rake clobber
     - rake configure_serviceaccount_ci_restore
     - rake
+    - rake test_preferences
+    - rake test_flowmanager
   after_script:
     # Clean up even if something failed.
     - cd gcp/live/dev
@@ -120,28 +122,32 @@ aws-stg:
   stage: stg
   tags:
     - aws
+  environment:
+    name: stg
   script:
     - cd aws/stg
     - rake clobber
     - rake
-  environment:
-    name: stg
 
 # DISABLED while we sort out the gcp-stg -> gcp-gpii-stg migration.
 .gcp-stg:
   stage: stg
   tags:
     - gcp
+  environment:
+    name: stg
   script:
     - cd gcp/live/stg
     - rake clobber
     - rake configure_serviceaccount_ci_restore
     - rake
-  environment:
-    name: stg
+    - rake test_preferences
+    - rake test_flowmanager
   after_script:
     # Clean up even if something failed.
     - cd gcp/live/stg
+    # Destroy Locust module (used for smoke tests)
+    - rake destroy_module[locust]
     # Remove all SA keys except current one to prevent hitting 10 keys per SA limit (GPII-3299)
     - rake destroy_sa_keys
 
@@ -182,12 +188,12 @@ aws-prd:
     RAKE_REALLY_RUN_IN_PRD: "true"
   tags:
     - aws
+  environment:
+    name: prd
   script:
     - cd aws/prd
     - rake clobber
     - rake
-  environment:
-    name: prd
 
 # DISABLED while we sort out the gcp-stg -> gcp-gpii-stg migration.
 .gcp-prd:
@@ -196,15 +202,19 @@ aws-prd:
     RAKE_REALLY_RUN_IN_PRD: "true"
   tags:
     - gcp
+  environment:
+    name: prd
   script:
     - cd gcp/live/prd
     - rake clobber
     - rake configure_serviceaccount_ci_restore
     - rake
-  environment:
-    name: prd
+    - rake test_preferences
+    - rake test_flowmanager
   after_script:
     # Clean up even if something failed.
     - cd gcp/live/prd
+    # Destroy Locust module (used for smoke tests)
+    - rake destroy_module[locust]
     # Remove all SA keys except current one to prevent hitting 10 keys per SA limit (GPII-3299)
     - rake destroy_sa_keys

common/charts/locust/Chart.yaml

@@ -0,0 +1,11 @@
+name: locust
+description: A modern load testing framework
+version: 0.3.0
+appVersion: 0.7.5
+maintainers:
+  - name: so0k
+    email: vincent.drl@gmail.com
+home: http://locust.io
+icon: https://pbs.twimg.com/profile_images/1867636195/locust-logo-orignal.png
+sources:
+  - https://github.com/honestbee/distributed-load-testing

common/charts/locust/README.md

@@ -0,0 +1,73 @@
+# Locust Helm Chart
+
+This is a templated deployment of [Locust](http://locust.io) for Distributed Load
+testing using Kubernetes.
+
+## Pre Requisites:
+
+* Requires (and tested with) helm `v2.1.2` or above.
+
+## Chart details
+
+This chart will do the following:
+
+* Convert all files in `tasks/` folder into a configmap
+* Create a Locust master and Locust worker deployment with the Target host
+  and Tasks file specified.
+
+
+### Installing the chart
+
+To install the chart with the release name `locust-nymph` in the default namespace:
+
+```bash
+helm install -n locust-nymph --set master.config.target-host=http://site.example.com stable/locust
+```
+
+| Parameter                    | Description                             | Default                                               |
+| ---------------------------- | ----------------------------------      | ----------------------------------------------------- |
+| `Name`                       | Locust master name                      | `locust`                                              |
+| `image.repository`           | Locust container image name             | `quay.io/honestbee/locust`                            |
+| `image.tag`                  | Locust Container image tag              | `0.7.5`                                               |
+| `image.pullSecrets`          | Locust Container image registry secret  | `None`                                                |
+| `service.type`               | k8s service type exposing master        | `NodePort`                                            |
+| `service.nodePort`           | Port on cluster to expose master        | `0`                                                   |
+| `service.annotations`        | KV containing custom annotations        | `{}`                                                  |
+| `service.extraLabels`        | KV containing extra labels              | `{}`                                                  |
+| `master.config.target-host`  | locust target host                      | `http://site.example.com`                             |
+| `worker.config.locust-script`| locust script to run                    | `/locust-tasks/tasks.py`                              |
+| `worker.replicaCount`        | Number of workers to run                | `2`                                                   |
+
+Specify parameters using `--set key=value[,key=value]` argument to `helm install`
+
+Alternatively a YAML file that specifies the values for the parameters can be provided like this:
+
+```bash
+$ helm install --name my-release -f values.yaml stable/locust
+```
+
+You can start the swarm from the command line using Port forwarding as follows:
+
+Get the Locust URL following the Post Installation notes.
+
+for example:
+```bash
+export LOCUST_URL=http://127.0.0.1:8089
+```
+
+Start / Monitor & Stop the Locust swarm via the web panel or with following commands:
+
+Start:
+```bash
+curl -XPOST $LOCUST_URL/swarm -d"locust_count=100&hatch_rate=10"
+```
+
+Monitor:
+```bash
+watch -n 1 "curl -s $LOCUST_URL/stats/requests | jq -r '[.user_count, .total_rps, .state] | @tsv'"
+```
+
+Stop:
+```bash
+curl $LOCUST_URL/stop
+```

common/charts/locust/templates/NOTES.txt

@@ -0,0 +1,24 @@
+{{ .Chart.Name }} installed!
+
+Get the Locust URL to visit by running these commands in the same shell:
+
+{{- if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get svc -n {{ .Release.Namespace }} {{ template "locust.master-svc" . }} -o jsonpath='{.spec.ports[?(@.name=="{{ .Values.service.name }}")].nodePort}')
+  export NODE_IP=$(kubectl get no -o jsonpath="{.items[0].status.addresses[0].address}")
+  export LOCUST_URL=http://$NODE_IP:$NODE_PORT/
+
+{{- else if contains "LoadBalancer" .Values.service.type }}
+  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+        You can watch the status by running 'kubectl get svc -n {{ .Release.Namespace }} -w {{ template "locust.master-svc" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "locust.master-svc" . }} --template "{{ "{{ range (index .status.loadBalancer.ingress 0) }}{{ . }}{{ end }}" }}")
+  export LOCUST_URL=http://$SERVICE_IP:{{ .Values.service.externalPort }}/
+
+{{- else if contains "ClusterIP"  .Values.service.type }}
+  export POD_NAME=$(kubectl get po --namespace {{ .Release.Namespace }} -l "app={{ template "locust.fullname" . }},component=master" -o jsonpath="{.items[0].metadata.name}")
+  export LOCUST_URL=http://127.0.0.1:{{ .Values.service.externalPort }}
+  kubectl port-forward $POD_NAME {{ .Values.service.externalPort  }}:{{ .Values.service.externalPort }} &
+
+{{- end }}
+
+For more information on Distributed load testing on Kubernetes using Locust, visit:
+https://cloud.google.com/solutions/distributed-load-testing-using-kubernetes

common/charts/locust/templates/_helpers.tpl

@@ -0,0 +1,27 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "locust.fullname" -}}
+{{- printf "%s-%s" .Release.Name "locust" | trunc 63 -}}
+{{- end -}}
+
+{{- define "locust.master-svc" -}}
+{{- printf "%s-%s" .Release.Name "master-svc" | trunc 63 -}}
+{{- end -}}
+
+{{- define "locust.master" -}}
+{{- printf "%s-%s" .Release.Name "master" | trunc 63 -}}
+{{- end -}}
+
+{{- define "locust.worker" -}}
+{{- printf "%s-%s" .Release.Name "worker" | trunc 63 -}}
+{{- end -}}
+
+{{/*
+Create fully qualified configmap name.
+*/}}
+{{- define "locust.worker-configmap" -}}
+{{- printf "%s-%s" .Release.Name "worker" -}}
+{{- end -}}

common/charts/locust/templates/master-deploy.yaml

@@ -0,0 +1,71 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: {{ template "locust.master" . }}
+  labels:
+    app: {{ template "locust.fullname" . }}
+    heritage: {{ .Release.Service | quote }}
+    release: {{ .Release.Name | quote }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    component: master
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: {{ default 1 .Values.master.maxSurge }}
+      maxUnavailable: {{ default 1 .Values.master.maxUnavailable }}
+  template:
+    metadata:
+      labels:
+        release: {{ .Release.Name | quote }}
+        app: {{ template "locust.fullname" . }}
+        component: "master"
+    spec:
+    {{- if .Values.image.pullSecrets }}
+      imagePullSecrets:
+{{ toYaml .Values.image.pullSecrets | indent 8 }}
+    {{- end }}
+      containers:
+      - name: locust
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        resources:
+{{ toYaml .Values.worker.resources | indent 10 }}
+        env:
+        {{- range $key, $value :=  .Values.master.config }}
+        - name: {{ $key | upper | replace "-" "_" }}
+          value: {{ $value | quote }}
+        {{- end }}
+        - name: LOCUST_MODE
+          value: "master"
+        - name: LOCUST_SCRIPT
+          value: {{ index .Values.worker.config "locust-script" | quote }}
+        ports:
+        - containerPort: {{ .Values.service.internalPort }}
+          name: loc-master-web
+          protocol: TCP
+        - containerPort: 5557
+          name: loc-master-p1
+          protocol: TCP
+        - containerPort: 5558
+          name: loc-master-p2
+          protocol: TCP
+        volumeMounts:
+          - name: locust-tasks
+            mountPath: /locust-tasks/
+        livenessProbe:
+          periodSeconds: 30
+          httpGet:
+            path: /
+            port: {{ .Values.service.internalPort }}
+        readinessProbe:
+          periodSeconds: 30
+          httpGet:
+            path: /
+            port: {{ .Values.service.internalPort }}
+      volumes:
+      - name: "locust-tasks"
+        configMap:
+          name: {{ template "locust.worker-configmap" . }}
+      restartPolicy: Always

common/charts/locust/templates/master-svc.yaml

@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "locust.master-svc" . }}
+  labels:
+    heritage: {{ .Release.Service | quote }}
+    release: {{ .Release.Name | quote }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    app: {{ template "locust.fullname" . }}
+    component: "master"
+    {{- range $key, $value :=  .Values.service.extraLabels }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+  {{- if .Values.service.annotations }}
+  annotations:
+  {{- range $key, $value :=  .Values.service.annotations }}
+    {{ $key }}: {{ $value | quote }}
+  {{- end }}
+  {{- end }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+  - name: {{ .Values.service.name }}
+    port: {{ .Values.service.externalPort }}
+    targetPort: {{ .Values.service.internalPort }}
+    nodePort: {{ .Values.service.nodePort }}
+    protocol: TCP
+  - name: master-p1
+    port: 5557
+    protocol: TCP
+    targetPort: 5557
+  - name: master-p2
+    port: 5558
+    protocol: TCP
+    targetPort: 5558
+  selector:
+    app: {{ template "locust.fullname" . }}
+    component: "master"
+  sessionAffinity: None

common/charts/locust/templates/worker-cm.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "locust.worker-configmap" . }}
+  labels:
+    heritage: {{ .Release.Service | quote }}
+    release: {{ .Release.Name | quote }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    app: {{ template "locust.fullname" . }}
+data:
+{{ (.Files.Glob "tasks/*").AsConfig | indent 2 }}