From e09954427abf2e19e72127df129217275f88863e Mon Sep 17 00:00:00 2001 From: Henry Kupty Date: Tue, 30 Jun 2020 23:18:27 +0200 Subject: [PATCH 1/9] Initial implementation of json-schema import This allows a very basic json-schema->malli conversion and will most likely need to be adequated to: - cleanup TODOS; - ensure function names are sane; - ensure all json-schema features are supported (at least for one specific version, i.e. draft-7). --- src/malli/json_schema/import.cljc | 110 ++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 src/malli/json_schema/import.cljc diff --git a/src/malli/json_schema/import.cljc b/src/malli/json_schema/import.cljc new file mode 100644 index 000000000..b7f0a92d5 --- /dev/null +++ b/src/malli/json_schema/import.cljc @@ -0,0 +1,110 @@ +(ns malli.json-schema.import + (:require [malli.core :as m])) + +(defmulti type->malli :type) + +(defn properties->malli [{:keys [required]} [k v]] + (cond-> [k] + (nil? (required k)) (conj {:optional true}) + true (conj (type->malli v)))) + +(defn $ref [v] + ;; TODO to be improved + (keyword (last (str/split v #"/")))) + +(defn v->malli [v] + (if-let [ref- (:$ref v)] + ($ref ref-) + (let [required (into #{} + (map keyword) + (:required v)) + closed? (false? (:additionalProperties v))] + (m/schema (cond-> [:map] + closed? (conj {:closed :true}) + true (into + (map (partial properties->malli {:required required})) + (:properties v))))))) + +(defn kv->malli [[k v]] + [k (v->malli v)]) + +(defmethod type->malli "string" [p] string?) +(defmethod type->malli "integer" [p] int?) +(defmethod type->malli "number" [p] number?) +(defmethod type->malli "boolean" [p] boolean?) +(defmethod type->malli "null" [p] nil?) +(defmethod type->malli "object" [p] (v->malli p)) +(defmethod type->malli "array" [p] (let [items (:items p)] + (cond + (vector? items) (into [:tuple] + (map type->malli) + items) + (map? items) [:vector (type->malli items)] + :else (throw (ex-info "Can't produce malli schema" {:p p}))))) + +(defn parse-top-level [js-schema] + (let [keys- (set (keys js-schema))] + (cond + (keys- :oneOf) (into [:or] + (map v->malli) + (:oneOf js-schema))))) + +(defn ->malli [obj] + [:schema {:registry (into {} + (map kv->malli) + (:definitions obj))} + (parse-top-level (dissoc obj :definitions))]) + +(defn properties->malli [{:keys [required]} [k v]] + (cond-> [k] + (nil? (required k)) (conj {:optional true}) + true (conj (type->malli v)))) + +(defn $ref [v] + ;; TODO to be improved + (keyword (last (str/split v #"/")))) + +(defn v->malli [v] + (if-let [ref- (:$ref v)] + ($ref ref-) + (let [required (into #{} + (map keyword) + (:required v)) + closed? (false? (:additionalProperties v))] + (m/schema (cond-> [:map] + closed? (conj {:closed :true}) + true (into + (map (partial properties->malli {:required required})) + (:properties v))))))) + +(defn kv->malli [[k v]] + [k (v->malli v)]) + +(defmethod type->malli "string" [p] string?) +(defmethod type->malli "integer" [p] int?) +(defmethod type->malli "number" [p] + ;; TODO support decimal/double + number?) +(defmethod type->malli "boolean" [p] boolean?) +(defmethod type->malli "null" [p] nil?) +(defmethod type->malli "object" [p] (v->malli p)) +(defmethod type->malli "array" [p] (let [items (:items p)] + (cond + (vector? items) (into [:tuple] + (map type->malli) + items) + (map? items) [:vector (type->malli items)] + :else (throw (ex-info "Can't produce malli schema" {:p p}))))) + +(defn parse-top-level [js-schema] + (let [keys- (set (keys js-schema))] + (cond + (keys- :oneOf) (into [:or] + (map v->malli) + (:oneOf js-schema))))) + +(defn ->malli [obj] + [:schema {:registry (into {} + (map kv->malli) + (:definitions obj))} + (parse-top-level (dissoc obj :definitions))]) From 74f62cf4afd6e087f520c9e639e5dc16db5eb5b8 Mon Sep 17 00:00:00 2001 From: Henry Kupty Date: Thu, 2 Jul 2020 17:06:48 +0200 Subject: [PATCH 2/9] Simplify functions, rename and make code simpler Also, remove duplication. --- src/malli/json_schema/import.cljc | 80 ++++++------------------------- 1 file changed, 15 insertions(+), 65 deletions(-) diff --git a/src/malli/json_schema/import.cljc b/src/malli/json_schema/import.cljc index b7f0a92d5..a4bf6d067 100644 --- a/src/malli/json_schema/import.cljc +++ b/src/malli/json_schema/import.cljc @@ -12,10 +12,8 @@ ;; TODO to be improved (keyword (last (str/split v #"/")))) -(defn v->malli [v] - (if-let [ref- (:$ref v)] - ($ref ref-) - (let [required (into #{} +(defn object->malli [v] + (let [required (into #{} (map keyword) (:required v)) closed? (false? (:additionalProperties v))] @@ -23,63 +21,11 @@ closed? (conj {:closed :true}) true (into (map (partial properties->malli {:required required})) - (:properties v))))))) - -(defn kv->malli [[k v]] - [k (v->malli v)]) - -(defmethod type->malli "string" [p] string?) -(defmethod type->malli "integer" [p] int?) -(defmethod type->malli "number" [p] number?) -(defmethod type->malli "boolean" [p] boolean?) -(defmethod type->malli "null" [p] nil?) -(defmethod type->malli "object" [p] (v->malli p)) -(defmethod type->malli "array" [p] (let [items (:items p)] - (cond - (vector? items) (into [:tuple] - (map type->malli) - items) - (map? items) [:vector (type->malli items)] - :else (throw (ex-info "Can't produce malli schema" {:p p}))))) - -(defn parse-top-level [js-schema] - (let [keys- (set (keys js-schema))] - (cond - (keys- :oneOf) (into [:or] - (map v->malli) - (:oneOf js-schema))))) - -(defn ->malli [obj] - [:schema {:registry (into {} - (map kv->malli) - (:definitions obj))} - (parse-top-level (dissoc obj :definitions))]) - -(defn properties->malli [{:keys [required]} [k v]] - (cond-> [k] - (nil? (required k)) (conj {:optional true}) - true (conj (type->malli v)))) - -(defn $ref [v] - ;; TODO to be improved - (keyword (last (str/split v #"/")))) - -(defn v->malli [v] - (if-let [ref- (:$ref v)] - ($ref ref-) - (let [required (into #{} - (map keyword) - (:required v)) - closed? (false? (:additionalProperties v))] - (m/schema (cond-> [:map] - closed? (conj {:closed :true}) - true (into - (map (partial properties->malli {:required required})) - (:properties v))))))) - -(defn kv->malli [[k v]] - [k (v->malli v)]) + (:properties v)))))) +(defmethod type->malli nil [p] + (when-let [-ref (:$ref v)] + ($ref -ref)) (defmethod type->malli "string" [p] string?) (defmethod type->malli "integer" [p] int?) (defmethod type->malli "number" [p] @@ -87,7 +33,7 @@ number?) (defmethod type->malli "boolean" [p] boolean?) (defmethod type->malli "null" [p] nil?) -(defmethod type->malli "object" [p] (v->malli p)) +(defmethod type->malli "object" [p] (object->malli p)) (defmethod type->malli "array" [p] (let [items (:items p)] (cond (vector? items) (into [:tuple] @@ -96,15 +42,19 @@ (map? items) [:vector (type->malli items)] :else (throw (ex-info "Can't produce malli schema" {:p p}))))) -(defn parse-top-level [js-schema] +(defn- parse-top-level [js-schema] (let [keys- (set (keys js-schema))] (cond (keys- :oneOf) (into [:or] - (map v->malli) + (map type->malli) (:oneOf js-schema))))) -(defn ->malli [obj] +(defn- map-values + ([-fn] (map (fn [[k v]] [k (-fn v)]))) + ([-fn coll] (sequence (map-values -fn) coll))) + +(defn json-schema-document->malli [obj] [:schema {:registry (into {} - (map kv->malli) + (map-values type->malli) (:definitions obj))} (parse-top-level (dissoc obj :definitions))]) From 9690651720e3d63768d81a206a8024822ed7a552 Mon Sep 17 00:00:00 2001 From: Henry Kupty Date: Thu, 2 Jul 2020 21:03:12 +0200 Subject: [PATCH 3/9] Add support for anyOf and allOf --- src/malli/json_schema/import.cljc | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/malli/json_schema/import.cljc b/src/malli/json_schema/import.cljc index a4bf6d067..f3c90e11f 100644 --- a/src/malli/json_schema/import.cljc +++ b/src/malli/json_schema/import.cljc @@ -43,11 +43,23 @@ :else (throw (ex-info "Can't produce malli schema" {:p p}))))) (defn- parse-top-level [js-schema] - (let [keys- (set (keys js-schema))] + (let [-keys (keys js-schema) + -key (first -keys)] + (when (pos? (dec (count -keys))) + (throw (ex-info "Invalid declaration" + {:number-of-keys (count -keys) + :schema js-schema}))) (cond - (keys- :oneOf) (into [:or] - (map type->malli) - (:oneOf js-schema))))) + (#{:oneOf :anyOf} -key) (into + ;; TODO Split :oneOf from :anyOf and figure out + ;; how to make it exclusively select o schema + [:or] + (map type->malli) + (:oneOf js-schema)) + (= :allOf -key) (into + [:and] + (map type->malli) + (:oneOf js-schema))))) (defn- map-values ([-fn] (map (fn [[k v]] [k (-fn v)]))) From b3f6a9a01a0ca234d76e1a969b50c2b0cdcd3889 Mon Sep 17 00:00:00 2001 From: Henry Kupty Date: Fri, 3 Jul 2020 22:12:42 +0200 Subject: [PATCH 4/9] Rename import->parse Also, reorganize functions to simplify processing, reduce duplication and avoid special casing. Lastly, make schema->malli the first entry point since anything can be a schema. It then delegates to types, aggregates or `$ref`s. --- .../json_schema/{import.cljc => parse.cljc} | 87 +++++++++++-------- 1 file changed, 50 insertions(+), 37 deletions(-) rename src/malli/json_schema/{import.cljc => parse.cljc} (57%) diff --git a/src/malli/json_schema/import.cljc b/src/malli/json_schema/parse.cljc similarity index 57% rename from src/malli/json_schema/import.cljc rename to src/malli/json_schema/parse.cljc index f3c90e11f..0fe1f8cbf 100644 --- a/src/malli/json_schema/import.cljc +++ b/src/malli/json_schema/parse.cljc @@ -1,17 +1,56 @@ -(ns malli.json-schema.import - (:require [malli.core :as m])) +(ns malli.json-schema.parse + (:require [malli.core :as m] + [clojure.string :as str])) -(defmulti type->malli :type) +;; Utility Functions +(defn- map-values + ([-fn] (map (fn [[k v]] [k (-fn v)]))) + ([-fn coll] (sequence (map-values -fn) coll))) -(defn properties->malli [{:keys [required]} [k v]] - (cond-> [k] - (nil? (required k)) (conj {:optional true}) - true (conj (type->malli v)))) +;; Parsing +(defmulti type->malli :type) (defn $ref [v] ;; TODO to be improved (keyword (last (str/split v #"/")))) +(defn schema->malli [js-schema] + (let [-keys (set (keys js-schema))] + (cond + (-keys :type) (type->malli js-schema) + + ;; Aggregates + (-keys :oneOf) + (into + ;; TODO Figure out how to make it exclusively select o schema + [:or] + (map schema->malli) + (:oneOf js-schema)) + + (-keys :anyOf) + (into + [:or] + (map schema->malli) + (:anyOf js-schema)) + + (-keys :allOf) + (into + [:and] + (map schema->malli) + (:allOf js-schema)) + + (-keys :not) [:not (schema->malli (:not js-schema))] + + (-keys :$ref) ($ref (:$ref js-schema)) + + :else + (throw (ex-info "Not supported" {:js-schema js-schema}))))) + +(defn properties->malli [{:keys [required]} [k v]] + (cond-> [k] + (nil? (required k)) (conj {:optional true}) + true (conj (schema->malli v)))) + (defn object->malli [v] (let [required (into #{} (map keyword) @@ -23,9 +62,6 @@ (map (partial properties->malli {:required required})) (:properties v)))))) -(defmethod type->malli nil [p] - (when-let [-ref (:$ref v)] - ($ref -ref)) (defmethod type->malli "string" [p] string?) (defmethod type->malli "integer" [p] int?) (defmethod type->malli "number" [p] @@ -37,36 +73,13 @@ (defmethod type->malli "array" [p] (let [items (:items p)] (cond (vector? items) (into [:tuple] - (map type->malli) + (map schema->malli) items) - (map? items) [:vector (type->malli items)] + (map? items) [:vector (schema->malli items)] :else (throw (ex-info "Can't produce malli schema" {:p p}))))) -(defn- parse-top-level [js-schema] - (let [-keys (keys js-schema) - -key (first -keys)] - (when (pos? (dec (count -keys))) - (throw (ex-info "Invalid declaration" - {:number-of-keys (count -keys) - :schema js-schema}))) - (cond - (#{:oneOf :anyOf} -key) (into - ;; TODO Split :oneOf from :anyOf and figure out - ;; how to make it exclusively select o schema - [:or] - (map type->malli) - (:oneOf js-schema)) - (= :allOf -key) (into - [:and] - (map type->malli) - (:oneOf js-schema))))) - -(defn- map-values - ([-fn] (map (fn [[k v]] [k (-fn v)]))) - ([-fn coll] (sequence (map-values -fn) coll))) - (defn json-schema-document->malli [obj] [:schema {:registry (into {} - (map-values type->malli) + (map-values schema->malli) (:definitions obj))} - (parse-top-level (dissoc obj :definitions))]) + (schema->malli obj)]) From 0ad4ca57c1cfcccebf90b407d6861f4625de054e Mon Sep 17 00:00:00 2001 From: Henry Kupty Date: Sat, 4 Jul 2020 06:20:28 +0200 Subject: [PATCH 5/9] Implement properties for strings, some int checks Also, adds top-level enums (and consts) and prepares for better range checks in int/number. --- src/malli/json_schema/parse.cljc | 45 +++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/src/malli/json_schema/parse.cljc b/src/malli/json_schema/parse.cljc index 0fe1f8cbf..61534e38c 100644 --- a/src/malli/json_schema/parse.cljc +++ b/src/malli/json_schema/parse.cljc @@ -1,5 +1,6 @@ (ns malli.json-schema.parse (:require [malli.core :as m] + [malli.util :as mu] [clojure.string :as str])) ;; Utility Functions @@ -19,6 +20,11 @@ (cond (-keys :type) (type->malli js-schema) + (-keys :enum) (into [:enum] + (:enum js-schema)) + + (-keys :const) [:enum (:const js-schema)] + ;; Aggregates (-keys :oneOf) (into @@ -44,7 +50,8 @@ (-keys :$ref) ($ref (:$ref js-schema)) :else - (throw (ex-info "Not supported" {:js-schema js-schema}))))) + (throw (ex-info "Not supported" {:json-schema js-schema + :reason ::schema-type}))))) (defn properties->malli [{:keys [required]} [k v]] (cond-> [k] @@ -53,6 +60,7 @@ (defn object->malli [v] (let [required (into #{} + ;; TODO Should use the same fn as $ref (map keyword) (:required v)) closed? (false? (:additionalProperties v))] @@ -62,11 +70,33 @@ (map (partial properties->malli {:required required})) (:properties v)))))) -(defmethod type->malli "string" [p] string?) -(defmethod type->malli "integer" [p] int?) -(defmethod type->malli "number" [p] - ;; TODO support decimal/double - number?) +(defmethod type->malli "string" [{:keys [pattern minLength maxLength enum]}] + ;; `format` metadata is deliberately not considered. + ;; String enums are stricter, so they're also implemented here. + (cond + pattern [:re pattern] + enum [:and + :string + (into [:enum] enum)] + [:string (cond-> {} + minLength (assoc :min minLength) + maxLength (assoc :max maxLength))])) + +(defmethod type->malli "integer" [{:keys [minimum maximum exclusiveMinimum exclusiveMaximum multipleOf] + :or {minimum Integer/MIN_VALUE + maximum Integer/MAX_VALUE}}] + ;; On draft 4, exclusive{Minimum,Maximum} is a boolean. + ;; TODO Decide on whether draft 4 will be supported + ;; TODO Implement exclusive{Minimum,Maximum} support + ;; TODO Implement multipleOf support + ;; TODO Wrap, when it makes sense, the values below with range checkers, i.e. [:< maximum] + ;; TODO extract ranges logic and reuse with number + (cond + (pos? minimum) pos-int? + (neg? maximum) neg-int? + :else int?)) + +(defmethod type->malli "number" [p] number?) (defmethod type->malli "boolean" [p] boolean?) (defmethod type->malli "null" [p] nil?) (defmethod type->malli "object" [p] (object->malli p)) @@ -76,7 +106,8 @@ (map schema->malli) items) (map? items) [:vector (schema->malli items)] - :else (throw (ex-info "Can't produce malli schema" {:p p}))))) + :else (throw (ex-info "Not Supported" {:json-schema p + :reason ::array-items}))))) (defn json-schema-document->malli [obj] [:schema {:registry (into {} From 404e664f04ada9ca39fa72973d45c4a8dda587f8 Mon Sep 17 00:00:00 2001 From: Henry Kupty Date: Thu, 17 Sep 2020 21:36:37 +0200 Subject: [PATCH 6/9] Fix missing `:else` clause --- src/malli/json_schema/parse.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/malli/json_schema/parse.cljc b/src/malli/json_schema/parse.cljc index 61534e38c..c13293aa1 100644 --- a/src/malli/json_schema/parse.cljc +++ b/src/malli/json_schema/parse.cljc @@ -78,7 +78,7 @@ enum [:and :string (into [:enum] enum)] - [:string (cond-> {} + :else [:string (cond-> {} minLength (assoc :min minLength) maxLength (assoc :max maxLength))])) From fc5059b0886c20eb2497f3244b9a79eeec71d9d2 Mon Sep 17 00:00:00 2001 From: Henry Kupty Date: Thu, 17 Sep 2020 21:37:17 +0200 Subject: [PATCH 7/9] Add annotations as malli properties --- src/malli/json_schema/parse.cljc | 61 ++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/src/malli/json_schema/parse.cljc b/src/malli/json_schema/parse.cljc index c13293aa1..f1fe83a93 100644 --- a/src/malli/json_schema/parse.cljc +++ b/src/malli/json_schema/parse.cljc @@ -1,8 +1,16 @@ (ns malli.json-schema.parse (:require [malli.core :as m] [malli.util :as mu] + [clojure.set :as set] [clojure.string :as str])) +(def annotations #{:title :description :default :examples}) + +(defn annotations->properties [js-schema] + (-> js-schema + (select-keys annotations) + (set/rename-keys {:examples :json-schema/examples}))) + ;; Utility Functions (defn- map-values ([-fn] (map (fn [[k v]] [k (-fn v)]))) @@ -17,41 +25,40 @@ (defn schema->malli [js-schema] (let [-keys (set (keys js-schema))] - (cond - (-keys :type) (type->malli js-schema) + (mu/update-properties + (cond + (-keys :type) (type->malli js-schema) - (-keys :enum) (into [:enum] - (:enum js-schema)) + (-keys :enum) (into [:enum] + (:enum js-schema)) - (-keys :const) [:enum (:const js-schema)] + (-keys :const) [:enum (:const js-schema)] - ;; Aggregates - (-keys :oneOf) - (into - ;; TODO Figure out how to make it exclusively select o schema - [:or] - (map schema->malli) - (:oneOf js-schema)) + ;; Aggregates + (-keys :oneOf) (into + ;; TODO Figure out how to make it exclusively select o schema + [:or] + (map schema->malli) + (:oneOf js-schema)) - (-keys :anyOf) - (into - [:or] - (map schema->malli) - (:anyOf js-schema)) + (-keys :anyOf) (into + [:or] + (map schema->malli) + (:anyOf js-schema)) - (-keys :allOf) - (into - [:and] - (map schema->malli) - (:allOf js-schema)) + (-keys :allOf) (into + [:and] + (map schema->malli) + (:allOf js-schema)) - (-keys :not) [:not (schema->malli (:not js-schema))] + (-keys :not) [:not (schema->malli (:not js-schema))] - (-keys :$ref) ($ref (:$ref js-schema)) + (-keys :$ref) ($ref (:$ref js-schema)) - :else - (throw (ex-info "Not supported" {:json-schema js-schema - :reason ::schema-type}))))) + :else (throw (ex-info "Not supported" {:json-schema js-schema + :reason ::schema-type}))) + merge + (annotations->properties js-schema)))) (defn properties->malli [{:keys [required]} [k v]] (cond-> [k] From 18332a6f72813c5c57f19904dbad69a638b6f1d3 Mon Sep 17 00:00:00 2001 From: Henry Kupty Date: Wed, 6 Jan 2021 14:41:36 +0100 Subject: [PATCH 8/9] Add min/max properties --- src/malli/json_schema/parse.cljc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/malli/json_schema/parse.cljc b/src/malli/json_schema/parse.cljc index f1fe83a93..ef5ef1587 100644 --- a/src/malli/json_schema/parse.cljc +++ b/src/malli/json_schema/parse.cljc @@ -60,11 +60,15 @@ merge (annotations->properties js-schema)))) -(defn properties->malli [{:keys [required]} [k v]] +(defn properties->malli [required [k v]] (cond-> [k] (nil? (required k)) (conj {:optional true}) true (conj (schema->malli v)))) +(defn- prop-size [pred?] (fn [-map] (pred? (count (keys -map))))) +(defn- min-properties [-min] (prop-size (partial <= -min))) +(defn- max-properties [-max] (prop-size (partial >= -max))) + (defn object->malli [v] (let [required (into #{} ;; TODO Should use the same fn as $ref @@ -74,7 +78,7 @@ (m/schema (cond-> [:map] closed? (conj {:closed :true}) true (into - (map (partial properties->malli {:required required})) + (map (partial properties->malli required)) (:properties v)))))) (defmethod type->malli "string" [{:keys [pattern minLength maxLength enum]}] From 9f380a73b5e07c11efa3e7ef21a256dfb066b7e8 Mon Sep 17 00:00:00 2001 From: Henry Kupty Date: Wed, 6 Jan 2021 14:58:13 +0100 Subject: [PATCH 9/9] add min/max properties to object --- src/malli/json_schema/parse.cljc | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/malli/json_schema/parse.cljc b/src/malli/json_schema/parse.cljc index ef5ef1587..abeefef34 100644 --- a/src/malli/json_schema/parse.cljc +++ b/src/malli/json_schema/parse.cljc @@ -69,17 +69,33 @@ (defn- min-properties [-min] (prop-size (partial <= -min))) (defn- max-properties [-max] (prop-size (partial >= -max))) +(defn with-min-max-poperties-size [malli v] + (let [predicates [(some->> v + (:minProperties) + (min-properties) + (conj [:fn])) + (some->> v + (:maxProperties) + (max-properties) + (conj [:fn]))]] + (cond->> malli + (some some? predicates) + (conj (into [:and] + (filter some?) + predicates))))) + (defn object->malli [v] (let [required (into #{} ;; TODO Should use the same fn as $ref (map keyword) (:required v)) closed? (false? (:additionalProperties v))] - (m/schema (cond-> [:map] - closed? (conj {:closed :true}) - true (into - (map (partial properties->malli required)) - (:properties v)))))) + (m/schema (-> [:map] + (cond-> closed? (conj {:closed :true})) + (into + (map (partial properties->malli required)) + (:properties v)) + (with-min-max-poperties-size v))))) (defmethod type->malli "string" [{:keys [pattern minLength maxLength enum]}] ;; `format` metadata is deliberately not considered.