diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fb0700e4..7000ca4e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,7 @@ jobs: - 2.11.12 - 2.12.17 - 2.13.10 + - 3.3.0 test-coverage: runs-on: ubuntu-latest steps: @@ -45,6 +46,7 @@ jobs: sbt ++2.12.17 coverage test coverageReport bash <(curl -s https://codecov.io/bash) mimaReport: + needs: test runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.gitignore b/.gitignore index 836905ce4..3913864c6 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ sonatype.sbt BUILD target/ lib_managed/ +project/metals.sbt project/boot/ project/build/target/ project/plugins/target/ diff --git a/.scalafmt.conf b/.scalafmt.conf index c9f903c4f..815cc1b61 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,10 +1,19 @@ version=3.6.0 runner.dialect = scala212 fileOverride { - "glob:**/scala-2.13*/**" { + "glob:**/scala-3/**" { + runner.dialect = scala3 + } + "glob:**/scala-2*/**" { runner.dialect = scala213 } } + +project.excludeFilters = [ + algebird-test/src/main/scala-3/com/twitter/algebird/macros/ArbitraryCaseClassMacro.scala, + algebird-test/src/main/scala-3/com/twitter/algebird/macros/Cuber.scala, +] + maxColumn = 110 docstrings.style = Asterisk newlines.alwaysBeforeMultilineDef = false diff --git a/algebird-core/src/main/scala-2.11/AggregatorCompat.scala b/algebird-core/src/main/scala-2.11/AggregatorCompat.scala new file mode 100644 index 000000000..3148ca015 --- /dev/null +++ b/algebird-core/src/main/scala-2.11/AggregatorCompat.scala @@ -0,0 +1,42 @@ +package com.twitter.algebird + +trait AggregatorCompat { + implicit def applicative[I]: Applicative[({ type L[O] = Aggregator[I, _, O] })#L] = + new AggregatorApplicative[I] +} + +/** + * Aggregators are Applicatives, but this hides the middle type. If you need a join that does not hide the + * middle type use join on the trait, or GeneratedTupleAggregator.fromN + */ +class AggregatorApplicative[I] extends Applicative[({ type L[O] = Aggregator[I, _, O] })#L] { + override def map[T, U](mt: Aggregator[I, _, T])(fn: T => U): Aggregator[I, _, U] = + mt.andThenPresent(fn) + override def apply[T](v: T): Aggregator[I, _, T] = + Aggregator.const(v) + override def join[T, U](mt: Aggregator[I, _, T], mu: Aggregator[I, _, U]): Aggregator[I, _, (T, U)] = + mt.join(mu) + override def join[T1, T2, T3]( + m1: Aggregator[I, _, T1], + m2: Aggregator[I, _, T2], + m3: Aggregator[I, _, T3] + ): Aggregator[I, _, (T1, T2, T3)] = + GeneratedTupleAggregator.from3((m1, m2, m3)) + + override def join[T1, T2, T3, T4]( + m1: Aggregator[I, _, T1], + m2: Aggregator[I, _, T2], + m3: Aggregator[I, _, T3], + m4: Aggregator[I, _, T4] + ): Aggregator[I, _, (T1, T2, T3, T4)] = + GeneratedTupleAggregator.from4((m1, m2, m3, m4)) + + override def join[T1, T2, T3, T4, T5]( + m1: Aggregator[I, _, T1], + m2: Aggregator[I, _, T2], + m3: Aggregator[I, _, T3], + m4: Aggregator[I, _, T4], + m5: Aggregator[I, _, T5] + ): Aggregator[I, _, (T1, T2, T3, T4, T5)] = + GeneratedTupleAggregator.from5((m1, m2, m3, m4, m5)) +} diff --git a/algebird-core/src/main/scala/com/twitter/algebird/CountMinSketch.scala b/algebird-core/src/main/scala-2.11/CountMinSketch.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/CountMinSketch.scala rename to algebird-core/src/main/scala-2.11/CountMinSketch.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/DecayedVector.scala b/algebird-core/src/main/scala-2.11/DecayedVector.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/DecayedVector.scala rename to algebird-core/src/main/scala-2.11/DecayedVector.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/DecayingCMS.scala b/algebird-core/src/main/scala-2.11/DecayingCMS.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/DecayingCMS.scala rename to algebird-core/src/main/scala-2.11/DecayingCMS.scala diff --git a/algebird-core/src/main/scala-2.11/FoldCompat.scala b/algebird-core/src/main/scala-2.11/FoldCompat.scala new file mode 100644 index 000000000..2dcb99b78 --- /dev/null +++ b/algebird-core/src/main/scala-2.11/FoldCompat.scala @@ -0,0 +1,22 @@ +package com.twitter.algebird + +trait FoldApplicativeCompat { + implicit def applicative[I]: Applicative[Fold[I, *]] = + new FoldApplicative[I] +} + +/** + * Folds are Applicatives! + */ +class FoldApplicative[I] extends Applicative[Fold[I, *]] { + override def map[T, U](mt: Fold[I, T])(fn: T => U): Fold[I, U] = + mt.map(fn) + override def apply[T](v: T): Fold[I, T] = + Fold.const(v) + override def join[T, U](mt: Fold[I, T], mu: Fold[I, U]): Fold[I, (T, U)] = + mt.join(mu) + override def sequence[T](ms: Seq[Fold[I, T]]): Fold[I, Seq[T]] = + Fold.sequence(ms) + override def joinWith[T, U, V](mt: Fold[I, T], mu: Fold[I, U])(fn: (T, U) => V): Fold[I, V] = + mt.joinWith(mu)(fn) +} diff --git a/algebird-core/src/main/scala/com/twitter/algebird/Interval.scala b/algebird-core/src/main/scala-2.11/Interval.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/Interval.scala rename to algebird-core/src/main/scala-2.11/Interval.scala diff --git a/algebird-core/src/main/scala-2.11/InvariantAlgebrasCompat.scala b/algebird-core/src/main/scala-2.11/InvariantAlgebrasCompat.scala new file mode 100644 index 000000000..42bfda4b7 --- /dev/null +++ b/algebird-core/src/main/scala-2.11/InvariantAlgebrasCompat.scala @@ -0,0 +1,24 @@ +package com.twitter.algebird + +class InvariantSemigroup[T, U](val forward: T => U, val reverse: U => T)(implicit val semigroup: Semigroup[T]) + extends Semigroup[U] { + override def plus(l: U, r: U): U = + forward(semigroup.plus(reverse(l), reverse(r))) + override def sumOption(iter: TraversableOnce[U]): Option[U] = + semigroup.sumOption(iter.map(reverse)).map(forward) + + /* + * Note these work for the subclasses since in those cases semigroup + * will be the appropriate algebra. + */ + override val hashCode: Int = (forward, reverse, semigroup).hashCode + override def equals(that: Any): Boolean = + that match { + case r: InvariantSemigroup[_, _] => + (hashCode == r.hashCode) && + (forward == r.forward) && + (reverse == r.reverse) && + (semigroup == r.semigroup) + case _ => false + } +} diff --git a/algebird-core/src/main/scala-2.11/JMapMonoidsCompat.scala b/algebird-core/src/main/scala-2.11/JMapMonoidsCompat.scala new file mode 100644 index 000000000..07000626c --- /dev/null +++ b/algebird-core/src/main/scala-2.11/JMapMonoidsCompat.scala @@ -0,0 +1,60 @@ +package com.twitter.algebird +import java.lang.{ + Boolean => JBool, + Double => JDouble, + Float => JFloat, + Integer => JInt, + Long => JLong, + Short => JShort +} +import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} + +import scala.collection.JavaConverters._ + +/** + * Since maps are mutable, this always makes a full copy. Prefer scala immutable maps if you use scala + * immutable maps, this operation is much faster TODO extend this to Group, Ring + */ +class JMapMonoid[K, V: Semigroup] extends Monoid[JMap[K, V]] { + override lazy val zero: JHashMap[K, V] = new JHashMap[K, V](0) + + val nonZero: (V => Boolean) = implicitly[Semigroup[V]] match { + case mon: Monoid[_] => mon.isNonZero(_) + case _ => _ => true + } + + override def isNonZero(x: JMap[K, V]): Boolean = + !x.isEmpty && (implicitly[Semigroup[V]] match { + case mon: Monoid[_] => + x.values.asScala.exists(v => mon.isNonZero(v)) + case _ => true + }) + override def plus(x: JMap[K, V], y: JMap[K, V]): JHashMap[K, V] = { + val (big, small, bigOnLeft) = + if (x.size > y.size) { + (x, y, true) + } else { + (y, x, false) + } + val vsemi = implicitly[Semigroup[V]] + val result = new JHashMap[K, V](big.size + small.size) + result.putAll(big) + small.entrySet.asScala.foreach { kv => + val smallK = kv.getKey + val smallV = kv.getValue + if (big.containsKey(smallK)) { + val bigV = big.get(smallK) + val newV = + if (bigOnLeft) vsemi.plus(bigV, smallV) else vsemi.plus(smallV, bigV) + if (nonZero(newV)) + result.put(smallK, newV) + else + result.remove(smallK) + } else { + // No need to explicitly add with zero on V, just put in the small value + result.put(smallK, smallV) + } + } + result + } +} diff --git a/algebird-core/src/main/scala-2.11/MapAlgebraCompat.scala b/algebird-core/src/main/scala-2.11/MapAlgebraCompat.scala new file mode 100644 index 000000000..3dc6fe230 --- /dev/null +++ b/algebird-core/src/main/scala-2.11/MapAlgebraCompat.scala @@ -0,0 +1,75 @@ +package com.twitter.algebird + +import scala.collection.mutable.{Builder, Map => MMap} +import scala.collection.{Map => ScMap} +import scala.collection.compat._ + +abstract class GenericMapMonoid[K, V, M <: ScMap[K, V]](implicit val semigroup: Semigroup[V]) + extends Monoid[M] + with MapOperations[K, V, M] { + + val nonZero: (V => Boolean) = semigroup match { + case mon: Monoid[_] => mon.isNonZero(_) + case _ => _ => true + } + + override def isNonZero(x: M): Boolean = + !x.isEmpty && (semigroup match { + case mon: Monoid[_] => + x.valuesIterator.exists(v => mon.isNonZero(v)) + case _ => true + }) + + override def plus(x: M, y: M): M = { + // Scala maps can reuse internal structure, so don't copy just add into the bigger one: + // This really saves computation when adding lots of small maps into big ones (common) + val (big, small, bigOnLeft) = + if (x.size > y.size) { + (x, y, true) + } else { + (y, x, false) + } + small match { + // Mutable maps create new copies of the underlying data on add so don't use the + // handleImmutable method. + // Cannot have a None so 'get' is safe here. + case _: MMap[_, _] => sumOption(Seq(big, small)).get + case _ => handleImmutable(big, small, bigOnLeft) + } + } + + private def handleImmutable(big: M, small: M, bigOnLeft: Boolean) = + small.foldLeft(big) { (oldMap, kv) => + val newV = big + .get(kv._1) + .map { bigV => + if (bigOnLeft) + semigroup.plus(bigV, kv._2) + else + semigroup.plus(kv._2, bigV) + } + .getOrElse(kv._2) + if (nonZero(newV)) + add(oldMap, kv._1 -> newV) + else + remove(oldMap, kv._1) + } + override def sumOption(items: TraversableOnce[M]): Option[M] = + if (items.iterator.isEmpty) None + else { + val mutable = MMap[K, V]() + items.iterator.foreach { m => + m.foreach { case (k, v) => + val oldVOpt = mutable.get(k) + // sorry for the micro optimization here: avoiding a closure + val newV = + if (oldVOpt.isEmpty) v else Semigroup.plus(oldVOpt.get, v) + if (nonZero(newV)) + mutable.update(k, newV) + else + mutable.remove(k) + } + } + Some(fromMutable(mutable)) + } +} diff --git a/algebird-core/src/main/scala-2.11/ScanApplicativeCompat.scala b/algebird-core/src/main/scala-2.11/ScanApplicativeCompat.scala new file mode 100644 index 000000000..1781f3837 --- /dev/null +++ b/algebird-core/src/main/scala-2.11/ScanApplicativeCompat.scala @@ -0,0 +1,15 @@ +package com.twitter.algebird + +class ScanApplicative[I] extends Applicative[Scan[I, *]] { + override def map[T, U](mt: Scan[I, T])(fn: T => U): Scan[I, U] = + mt.andThenPresent(fn) + + override def apply[T](v: T): Scan[I, T] = + Scan.const(v) + + override def join[T, U](mt: Scan[I, T], mu: Scan[I, U]): Scan[I, (T, U)] = + mt.join(mu) +} +trait ScanApplicativeCompat { + implicit def applicative[I]: Applicative[Scan[I, *]] = new ScanApplicative[I] +} diff --git a/algebird-core/src/main/scala/com/twitter/algebird/SpaceSaver.scala b/algebird-core/src/main/scala-2.11/SpaceSaver.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/SpaceSaver.scala rename to algebird-core/src/main/scala-2.11/SpaceSaver.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/VectorSpace.scala b/algebird-core/src/main/scala-2.11/VectorSpace.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/VectorSpace.scala rename to algebird-core/src/main/scala-2.11/VectorSpace.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/monad/EitherMonad.scala b/algebird-core/src/main/scala-2.11/monad/EitherMonad.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/monad/EitherMonad.scala rename to algebird-core/src/main/scala-2.11/monad/EitherMonad.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/monad/Reader.scala b/algebird-core/src/main/scala-2.11/monad/Reader.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/monad/Reader.scala rename to algebird-core/src/main/scala-2.11/monad/Reader.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/monad/StateWithError.scala b/algebird-core/src/main/scala-2.11/monad/StateWithError.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/monad/StateWithError.scala rename to algebird-core/src/main/scala-2.11/monad/StateWithError.scala diff --git a/algebird-core/src/main/scala-2.12+/AggregatorCompat.scala b/algebird-core/src/main/scala-2.12+/AggregatorCompat.scala new file mode 100644 index 000000000..a4dd9df4c --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/AggregatorCompat.scala @@ -0,0 +1,42 @@ +package com.twitter.algebird + +private[algebird] trait AggregatorCompat { + implicit def applicative[I]: Applicative[({ type L[O] = Aggregator[I, ?, O] })#L] = + new AggregatorApplicative[I] +} + +/** + * Aggregators are Applicatives, but this hides the middle type. If you need a join that does not hide the + * middle type use join on the trait, or GeneratedTupleAggregator.fromN + */ +class AggregatorApplicative[I] extends Applicative[({ type L[O] = Aggregator[I, ?, O] })#L] { + override def map[T, U](mt: Aggregator[I, ?, T])(fn: T => U): Aggregator[I, ?, U] = + mt.andThenPresent(fn) + override def apply[T](v: T): Aggregator[I, ?, T] = + Aggregator.const(v) + override def join[T, U](mt: Aggregator[I, ?, T], mu: Aggregator[I, ?, U]): Aggregator[I, ?, (T, U)] = + mt.join(mu) + override def join[T1, T2, T3]( + m1: Aggregator[I, ?, T1], + m2: Aggregator[I, ?, T2], + m3: Aggregator[I, ?, T3] + ): Aggregator[I, ?, (T1, T2, T3)] = + GeneratedTupleAggregator.from3((m1, m2, m3)) + + override def join[T1, T2, T3, T4]( + m1: Aggregator[I, ?, T1], + m2: Aggregator[I, ?, T2], + m3: Aggregator[I, ?, T3], + m4: Aggregator[I, ?, T4] + ): Aggregator[I, ?, (T1, T2, T3, T4)] = + GeneratedTupleAggregator.from4((m1, m2, m3, m4)) + + override def join[T1, T2, T3, T4, T5]( + m1: Aggregator[I, ?, T1], + m2: Aggregator[I, ?, T2], + m3: Aggregator[I, ?, T3], + m4: Aggregator[I, ?, T4], + m5: Aggregator[I, ?, T5] + ): Aggregator[I, ?, (T1, T2, T3, T4, T5)] = + GeneratedTupleAggregator.from5((m1, m2, m3, m4, m5)) +} diff --git a/algebird-core/src/main/scala-2.12+/CountMinSketch.scala b/algebird-core/src/main/scala-2.12+/CountMinSketch.scala new file mode 100644 index 000000000..826aebd5a --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/CountMinSketch.scala @@ -0,0 +1,1420 @@ +/* +Copyright 2012 Twitter, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.algebird + +import algebra.CommutativeMonoid + +import scala.collection.compat._ + +/** + * A Count-Min sketch is a probabilistic data structure used for summarizing streams of data in sub-linear + * space. + * + * It works as follows. Let `(eps, delta)` be two parameters that describe the confidence in our error + * estimates, and let `d = ceil(ln 1/delta)` and `w = ceil(e / eps)`. + * + * Note: Throughout the code `d` and `w` are called `depth` and `width`, respectively. + * + * Then: + * + * - Take `d` pairwise independent hash functions `h_i`, each of which maps onto the domain `[0, w - 1]`. + * - Create a 2-dimensional table of counts, with `d` rows and `w` columns, initialized with all zeroes. + * - When a new element x arrives in the stream, update the table of counts by setting `counts[i, h_i[x]] += + * 1`, for each `1 <= i <= d`. + * - (Note the rough similarity to a Bloom filter.) + * + * As an example application, suppose you want to estimate the number of times an element `x` has appeared in + * a data stream so far. The Count-Min sketch estimate of this frequency is + * + * min_i { counts[i, h_i[x]] } + * + * With probability at least `1 - delta`, this estimate is within `eps * N` of the true frequency (i.e., `true + * frequency <= estimate <= true frequency + eps * N`), where N is the total size of the stream so far. + * + * See http://www.eecs.harvard.edu/~michaelm/CS222/countmin.pdf for technical details, including proofs of the + * estimates and error bounds used in this implementation. + * + * Parts of this implementation are taken from + * https://github.com/clearspring/stream-lib/blob/master/src/main/java/com/clearspring/analytics/stream/frequency/CountMinSketch.java + * + * @author + * Edwin Chen + */ +/** + * Monoid for adding CMS sketches. + * + * =Usage= + * + * `eps` and `delta` are parameters that bound the error of each query estimate. For example, errors in + * answering point queries (e.g., how often has element x appeared in the stream described by the sketch?) are + * often of the form: "with probability p >= 1 - delta, the estimate is close to the truth by some factor + * depending on eps." + * + * The type `K` is the type of items you want to count. You must provide an implicit `CMSHasher[K]` for `K`, + * and Algebird ships with several such implicits for commonly used types such as `Long` and `BigInt`. + * + * If your type `K` is not supported out of the box, you have two options: 1) You provide a "translation" + * function to convert items of your (unsupported) type `K` to a supported type such as Double, and then use + * the `contramap` function of [[CMSHasher]] to create the required `CMSHasher[K]` for your type (see the + * documentation of [[CMSHasher]] for an example); 2) You implement a `CMSHasher[K]` from scratch, using the + * existing CMSHasher implementations as a starting point. + * + * Note: Because Arrays in Scala/Java not have sane `equals` and `hashCode` implementations, you cannot safely + * use types such as `Array[Byte]`. Extra work is required for Arrays. For example, you may opt to convert + * `Array[T]` to a `Seq[T]` via `toSeq`, or you can provide appropriate wrapper classes. Algebird provides one + * such wrapper class, [[Bytes]], to safely wrap an `Array[Byte]` for use with CMS. + * + * @param eps + * One-sided error bound on the error of each point query, i.e. frequency estimate. + * @param delta + * A bound on the probability that a query estimate does not lie within some small interval (an interval + * that depends on `eps`) around the truth. + * @param seed + * A seed to initialize the random number generator used to create the pairwise independent hash functions. + * @param maxExactCountOpt + * An Option parameter about how many exact counts a sparse CMS wants to keep. + * @tparam K + * The type used to identify the elements to be counted. For example, if you want to count the occurrence of + * user names, you could map each username to a unique numeric ID expressed as a `Long`, and then count the + * occurrences of those `Long`s with a CMS of type `K=Long`. Note that this mapping between the elements of + * your problem domain and their identifiers used for counting via CMS should be bijective. We require a + * [[CMSHasher]] context bound for `K`, see [[CMSHasherImplicits]] for available implicits that can be + * imported. Which type K should you pick in practice? For domains that have less than `2^64` unique + * elements, you'd typically use `Long`. For larger domains you can try `BigInt`, for example. Other + * possibilities include Spire's `SafeLong` and `Numerical` data types (https://github.com/non/spire), + * though Algebird does not include the required implicits for CMS-hashing (cf. [[CMSHasherImplicits]]. + */ +class CMSMonoid[K: CMSHasher](eps: Double, delta: Double, seed: Int, maxExactCountOpt: Option[Int] = None) + extends Monoid[CMS[K]] + with CommutativeMonoid[CMS[K]] { + + val params: CMSParams[K] = { + val hashes: Seq[CMSHash[K]] = CMSFunctions.generateHashes(eps, delta, seed) + CMSParams(hashes, eps, delta, maxExactCountOpt) + } + + override val zero: CMS[K] = CMSZero[K](params) + + /** + * Combines the two sketches. + * + * The sketches must use the same hash functions. + */ + override def plus(left: CMS[K], right: CMS[K]): CMS[K] = { + require(left.params.hashes == right.params.hashes, "The sketches must use the same hash functions.") + left ++ right + } + + /** + * Creates a sketch out of a single item. + */ + def create(item: K): CMS[K] = CMSItem[K](item, 1L, params) + + /** + * Creates a sketch out of multiple items. + */ + def create(data: Seq[K]): CMS[K] = { + val summation = new CMSSummation(params) + data.foreach(k => summation.insert(k, 1L)) + summation.result + } + + override def sumOption(sketches: TraversableOnce[CMS[K]]): Option[CMS[K]] = + if (sketches.iterator.isEmpty) None else Some(sum(sketches)) + + override def sum(sketches: TraversableOnce[CMS[K]]): CMS[K] = { + val summation = new CMSSummation(params) + summation.updateAll(sketches) + summation.result + } +} + +/** + * This mutable builder can be used when speed is essential and you can be sure the scope of the mutability + * cannot escape in an unsafe way. The intended use is to allocate and call result in one method without + * letting a reference to the instance escape into a closure. + */ +class CMSSummation[K](params: CMSParams[K]) { + private[this] val hashes = params.hashes.toArray + private[this] val height = CMSFunctions.depth(params.delta) + private[this] val width = CMSFunctions.width(params.eps) + private[this] val cells = new Array[Long](height * width) + private[this] var totalCount = 0L + + final def insert(k: K, count: Long): Unit = { + var row = 0 + var offset = 0 + val hs = hashes + while (row < hs.length) { + cells(offset + hs(row)(k)) += count + offset += width + row += 1 + } + totalCount += count + } + + def updateAll(sketches: TraversableOnce[CMS[K]]): Unit = + sketches.iterator.foreach(updateInto) + + def updateInto(cms: CMS[K]): Unit = + cms match { + case CMSZero(_) => + () + case CMSItem(item, count, _) => + insert(item, count) + case SparseCMS(table, _, _) => + table.foreach { case (item, c) => + insert(item, c) + } + case CMSInstance(CMSInstance.CountsTable(matrix), count, _) => + var offset = 0 + val rit = matrix.iterator + while (rit.hasNext) { + var col = 0 + val cit = rit.next().iterator + while (cit.hasNext) { + cells(offset + col) += cit.next() + col += 1 + } + offset += width + } + totalCount += count + } + + def result: CMS[K] = + if (totalCount == 0L) CMSZero(params) + else { + def vectorize(row: Int): Vector[Long] = { + val offset = row * width + val b = Vector.newBuilder[Long] + var col = 0 + while (col < width) { + b += cells(offset + col) + col += 1 + } + b.result() + } + + val b = Vector.newBuilder[Vector[Long]] + var row = 0 + while (row < height) { + b += vectorize(row) + row += 1 + } + CMSInstance(CMSInstance.CountsTable(b.result()), totalCount, params) + } +} + +/** + * An Aggregator for [[CMS]]. Can be created using CMS.aggregator. + */ +case class CMSAggregator[K](cmsMonoid: CMSMonoid[K]) extends MonoidAggregator[K, CMS[K], CMS[K]] { + override val monoid: CMSMonoid[K] = cmsMonoid + + override def prepare(value: K): CMS[K] = monoid.create(value) + + override def present(cms: CMS[K]): CMS[K] = cms + +} + +/** + * Configuration parameters for [[CMS]]. + * + * @param hashes + * Pair-wise independent hashes functions. We need `N=depth` such functions (`depth` can be derived from + * `delta`). + * @param eps + * One-sided error bound on the error of each point query, i.e. frequency estimate. + * @param delta + * A bound on the probability that a query estimate does not lie within some small interval (an interval + * that depends on `eps`) around the truth. + * @param maxExactCountOpt + * An Option parameter about how many exact counts a sparse CMS wants to keep. + * @tparam K + * The type used to identify the elements to be counted. + */ +case class CMSParams[K]( + hashes: Seq[CMSHash[K]], + eps: Double, + delta: Double, + maxExactCountOpt: Option[Int] = None +) { + + require(0 < eps && eps < 1, "eps must lie in (0, 1)") + require(0 < delta && delta < 1, "delta must lie in (0, 1)") + require( + hashes.size >= CMSFunctions.depth(delta), + s"we require at least ${CMSFunctions.depth(delta)} hash functions" + ) + +} + +/** + * Helper functions to generate or to translate between various CMS parameters (cf. [[CMSParams]]). + */ +object CMSFunctions { + + /** + * Translates from `width` to `eps`. + */ + def eps(width: Int): Double = scala.math.exp(1.0) / width + + /** + * Translates from `depth` to `delta`. + */ + @throws[IllegalArgumentException]("if depth is too large, causing precision errors when computing delta") + def delta(depth: Int): Double = { + val i = scala.math.exp(-depth) + require( + i > 0.0, + s"depth must be smaller as it causes precision errors when computing delta ($depth led to an invalid delta of $i)" + ) + i + } + + /** + * Translates from `delta` to `depth`. + */ + @throws[IllegalArgumentException]("if delta is is not in (0, 1)") + def depth(delta: Double): Int = { + require(0 < delta && delta < 1, "delta must lie in (0, 1)") + scala.math.ceil(scala.math.log(1.0 / delta)).toInt + } + + /** + * Translates from `eps` to `width`. + */ + def width(eps: Double): Int = + scala.math.ceil(truncatePrecisionError(scala.math.exp(1) / eps)).toInt + + /** + * Compute maxExactCount from parameters or `depth` and `width` + */ + def maxExactCount(maxExactCountOpt: Option[Int], depth: Int, width: Int): Int = + maxExactCountOpt.getOrElse(math.max(width * depth / 100, 50)) + + // Eliminates precision errors such as the following: + // + // scala> val width = 39 + // scala> scala.math.exp(1) / CMSFunctions.eps(width) + // res171: Double = 39.00000000000001 <<< should be 39.0 + // + // Because of the actual types on which CMSFunctions operates (i.e. Int and Double), the maximum number of decimal + // places should be 6. + private def truncatePrecisionError(i: Double, decimalPlaces: Int = 6) = + BigDecimal(i) + .setScale(decimalPlaces, BigDecimal.RoundingMode.HALF_UP) + .toDouble + + /** + * Generates `N=depth` pair-wise independent hash functions. + * + * @param eps + * One-sided error bound on the error of each point query, i.e. frequency estimate. + * @param delta + * Error bound on the probability that a query estimate does NOT lie within some small interval around the + * truth. + * @param seed + * Seed for the random number generator. + * @tparam K + * The type used to identify the elements to be counted. + * @return + * The generated hash functions. + */ + def generateHashes[K: CMSHasher](eps: Double, delta: Double, seed: Int): Seq[CMSHash[K]] = { + // Typically, we would use d -- aka depth -- pair-wise independent hash functions of the form + // + // h_i(x) = a_i * x + b_i (mod p) + // + // But for this particular application, setting b_i does not matter (since all it does is shift the results of a + // particular hash), so we omit it (by setting b_i to 0) and simply use hash functions of the form + // + // h_i(x) = a_i * x (mod p) + // + val r = new scala.util.Random(seed) + val numHashes = depth(delta) + val numCounters = width(eps) + (0 to (numHashes - 1)).map(_ => CMSHash[K](r.nextInt(), 0, numCounters)) + } + +} + +/** + * A trait for CMS implementations that can count elements in a data stream and that can answer point queries + * (i.e. frequency estimates) for these elements. + * + * Known implementations: [[CMS]], [[TopCMS]]. + * + * @tparam K + * The type used to identify the elements to be counted. + * @tparam C + * The type of the actual CMS that implements this trait. + */ +trait CMSCounting[K, C[_]] { + + /** + * Returns the one-sided error bound on the error of each point query, i.e. frequency estimate. + */ + def eps: Double + + /** + * Returns the bound on the probability that a query estimate does NOT lie within some small interval (an + * interval that depends on `eps`) around the truth. + */ + def delta: Double + + /** + * Number of hash functions (also: number of rows in the counting table). This number is derived from + * `delta`. + */ + def depth: Int = CMSFunctions.depth(delta) + + /** + * Number of counters per hash function (also: number of columns in the counting table). This number is + * derived from `eps`. + */ + def width: Int = CMSFunctions.width(eps) + + /** + * An Option parameter about how many exact counts a sparse CMS wants to keep + */ + def maxExactCountOpt: Option[Int] + + /** + * Number of exact counts a sparse CMS wants to keep. This number is derived from `maxExactCountOpt`. + */ + def maxExactCount: Int = + CMSFunctions.maxExactCount(maxExactCountOpt, depth, width) + + /** + * Returns a new sketch that is the combination of this sketch and the other sketch. + */ + def ++(other: C[K]): C[K] + + /** + * Counts the item and returns the result as a new sketch. + */ + def +(item: K): C[K] = this + (item, 1L) + + /** + * Counts the item `count` times and returns the result as a new sketch. + */ + def +(item: K, count: Long): C[K] + + /** + * Returns an estimate of the total number of times this item has been seen in the stream so far. This + * estimate is an upper bound. + * + * It is always true that `estimatedFrequency >= trueFrequency`. With probability `p >= 1 - delta`, it also + * holds that `estimatedFrequency <= trueFrequency + eps * totalCount`. + */ + def frequency(item: K): Approximate[Long] + + /** + * Returns an estimate of the inner product against another data stream. + * + * In other words, let a_i denote the number of times element i has been seen in the data stream summarized + * by this CMS, and let b_i denote the same for the other CMS. Then this returns an estimate of ` = + * \sum a_i b_i`. + * + * Note: This can also be viewed as the join size between two relations. + * + * It is always true that actualInnerProduct <= estimatedInnerProduct. With probability `p >= 1 - delta`, it + * also holds that `estimatedInnerProduct <= actualInnerProduct + eps * thisTotalCount * otherTotalCount`. + */ + def innerProduct(other: C[K]): Approximate[Long] + + /** + * Total number of elements counted (i.e. seen in the data stream) so far. + */ + def totalCount: Long + + /** + * The first frequency moment is the total number of elements in the stream. + */ + def f1: Long = totalCount + + /** + * The second frequency moment is `\sum a_i^2`, where `a_i` is the count of the i-th element. + */ + def f2: Approximate[Long] + +} + +/** + * A trait for CMS implementations that can track heavy hitters in a data stream. + * + * It is up to the implementation how the semantics of tracking heavy hitters are defined. For instance, one + * implementation could track the "top %" heavy hitters whereas another implementation could track the "top N" + * heavy hitters. + * + * Known implementations: [[TopCMS]]. + * + * @tparam K + * The type used to identify the elements to be counted. + */ +trait CMSHeavyHitters[K] { + + /** + * The pluggable logic of how heavy hitters are being tracked. + */ + def heavyHittersLogic: HeavyHittersLogic[K] + + /** + * Returns the set of heavy hitters. + */ + def heavyHitters: Set[K] + +} + +object CMS { + + def monoid[K: CMSHasher](eps: Double, delta: Double, seed: Int): CMSMonoid[K] = + monoid(eps, delta, seed, None) + def monoid[K: CMSHasher]( + eps: Double, + delta: Double, + seed: Int, + maxExactCountOpt: Option[Int] + ): CMSMonoid[K] = + new CMSMonoid[K](eps, delta, seed, maxExactCountOpt) + + def monoid[K: CMSHasher](depth: Int, width: Int, seed: Int): CMSMonoid[K] = + monoid(depth, width, seed, None) + def monoid[K: CMSHasher](depth: Int, width: Int, seed: Int, maxExactCountOpt: Option[Int]): CMSMonoid[K] = + monoid(CMSFunctions.eps(width), CMSFunctions.delta(depth), seed, maxExactCountOpt) + + def aggregator[K: CMSHasher](eps: Double, delta: Double, seed: Int): CMSAggregator[K] = + aggregator(eps, delta, seed, None) + def aggregator[K: CMSHasher]( + eps: Double, + delta: Double, + seed: Int, + maxExactCountOpt: Option[Int] + ): CMSAggregator[K] = + new CMSAggregator[K](monoid(eps, delta, seed, maxExactCountOpt)) + + def aggregator[K: CMSHasher](depth: Int, width: Int, seed: Int): CMSAggregator[K] = + aggregator(depth, width, seed, None) + def aggregator[K: CMSHasher]( + depth: Int, + width: Int, + seed: Int, + maxExactCountOpt: Option[Int] + ): CMSAggregator[K] = + aggregator(CMSFunctions.eps(width), CMSFunctions.delta(depth), seed, maxExactCountOpt) + + /** + * Returns a fresh, zeroed CMS instance. + */ + def apply[K: CMSHasher]( + eps: Double, + delta: Double, + seed: Int, + maxExactCountOpt: Option[Int] = None + ): CMS[K] = { + val params = { + val hashes: Seq[CMSHash[K]] = + CMSFunctions.generateHashes(eps, delta, seed) + CMSParams(hashes, eps, delta, maxExactCountOpt) + } + CMSZero[K](params) + } + +} + +/** + * A Count-Min sketch data structure that allows for counting and frequency estimation of elements in a data + * stream. + * + * Tip: If you also need to track heavy hitters ("Top N" problems), take a look at [[TopCMS]]. + * + * =Usage= + * + * This example demonstrates how to count `Long` elements with [[CMS]], i.e. `K=Long`. + * + * Note that the actual counting is always performed with a `Long`, regardless of your choice of `K`. That is, + * the counting table behind the scenes is backed by `Long` values (at least in the current implementation), + * and thus the returned frequency estimates are always instances of `Approximate[Long]`. + * + * @example + * {{{ + * + * // Creates a monoid for a CMS that can count `Long` elements. val cmsMonoid: CMSMonoid[Long] = { val eps = + * 0.001 val delta = 1E-10 val seed = 1 CMS.monoid[Long](eps, delta, seed) } + * + * // Creates a CMS instance that has counted the element `1L`. val cms: CMS[Long] = cmsMonoid.create(1L) + * + * // Estimates the frequency of `1L` val estimate: Approximate[Long] = cms.frequency(1L) + * }}} + * + * @tparam K + * The type used to identify the elements to be counted. + */ +sealed abstract class CMS[K](val params: CMSParams[K]) extends java.io.Serializable with CMSCounting[K, CMS] { + + override val eps: Double = params.eps + + override val delta: Double = params.delta + + override val maxExactCountOpt: Option[Int] = params.maxExactCountOpt + + override def f2: Approximate[Long] = innerProduct(this) + +} + +/** + * Zero element. Used for initialization. + */ +case class CMSZero[K](override val params: CMSParams[K]) extends CMS[K](params) { + + override val totalCount: Long = 0L + + override def +(item: K, count: Long): CMS[K] = CMSItem[K](item, count, params) + + override def ++(other: CMS[K]): CMS[K] = other + + override def frequency(item: K): Approximate[Long] = Approximate.exact(0L) + + override def innerProduct(other: CMS[K]): Approximate[Long] = + Approximate.exact(0L) + +} + +/** + * Used for holding a single element, to avoid repeatedly adding elements from sparse counts tables. + */ +case class CMSItem[K](item: K, override val totalCount: Long, override val params: CMSParams[K]) + extends CMS[K](params) { + + override def +(x: K, count: Long): CMS[K] = + SparseCMS[K](params) + (item, totalCount) + (x, count) + + override def ++(other: CMS[K]): CMS[K] = + other match { + case _: CMSZero[?] => this + case other: CMSItem[K] => + CMSInstance[K](params) + (item, totalCount) + (other.item, other.totalCount) + case _ => other + item + } + + override def frequency(x: K): Approximate[Long] = + if (item == x) Approximate.exact(totalCount) else Approximate.exact(0L) + + override def innerProduct(other: CMS[K]): Approximate[Long] = + Approximate.exact(totalCount) * other.frequency(item) + +} + +/** + * A sparse Count-Min sketch structure, used for situations where the key is highly skewed. + */ +case class SparseCMS[K]( + exactCountTable: Map[K, Long], + override val totalCount: Long, + override val params: CMSParams[K] +) extends CMS[K](params) { + import SparseCMS._ + + override def +(x: K, count: Long): CMS[K] = { + val currentCount = exactCountTable.getOrElse(x, 0L) + val newTable = exactCountTable.updated(x, currentCount + count) + if (newTable.size < maxExactCount) { + // still sparse + SparseCMS(newTable, totalCount = totalCount + count, params = params) + } else { + toDense(newTable, params) + } + } + + override def ++(other: CMS[K]): CMS[K] = + other match { + case _: CMSZero[?] => this + case other: CMSItem[K] => this + (other.item, other.totalCount) + case other: SparseCMS[K] => + // This SparseCMS's maxExactCount is used, so ++ is not communitive + val newTable = Semigroup.plus(exactCountTable, other.exactCountTable) + if (newTable.size < maxExactCount) { + // still sparse + SparseCMS(newTable, totalCount = totalCount + other.totalCount, params = params) + } else { + toDense(newTable, params) + } + + case other: CMSInstance[K] => other ++ this + } + + override def frequency(x: K): Approximate[Long] = + Approximate.exact(exactCountTable.getOrElse(x, 0L)) + + override def innerProduct(other: CMS[K]): Approximate[Long] = + exactCountTable.iterator + .map { case (x, count) => Approximate.exact(count) * other.frequency(x) } + .reduceOption(_ + _) + .getOrElse(Approximate.exact(0L)) +} + +object SparseCMS { + + /** + * Creates a new [[SparseCMS]] with empty exactCountTable + */ + def apply[K](params: CMSParams[K]): SparseCMS[K] = { + val exactCountTable = Map[K, Long]() + SparseCMS[K](exactCountTable, totalCount = 0, params = params) + } + + /** + * Creates a new [[CMSInstance]] from a Map[K, Long] + */ + def toDense[K](exactCountTable: Map[K, Long], params: CMSParams[K]): CMS[K] = + // Create new CMSInstace + exactCountTable.foldLeft(CMSInstance[K](params)) { case (cms, (x, count)) => + cms + (x, count) + } +} + +/** + * The general Count-Min sketch structure, used for holding any number of elements. + */ +case class CMSInstance[K]( + countsTable: CMSInstance.CountsTable[K], + override val totalCount: Long, + override val params: CMSParams[K] +) extends CMS[K](params) { + + override def ++(other: CMS[K]): CMS[K] = + other match { + case _: CMSZero[?] => this + case other: CMSItem[K] => this + other.item + case other: SparseCMS[K] => + other.exactCountTable.foldLeft(this) { case (cms, (x, count)) => + cms + (x, count) + } + case other: CMSInstance[K] => + val newTable = countsTable ++ other.countsTable + val newTotalCount = totalCount + other.totalCount + CMSInstance[K](newTable, newTotalCount, params) + } + + private def makeApprox(est: Long): Approximate[Long] = + if (est == 0L) Approximate.exact(0L) + else { + val lower = math.max(0L, est - (eps * totalCount).toLong) + Approximate(lower, est, est, 1 - delta) + } + + override def frequency(item: K): Approximate[Long] = { + var freq = Long.MaxValue + val hs = params.hashes + val it = countsTable.counts.iterator + var i = 0 + while (it.hasNext) { + val row = it.next() + val count = row(hs(i)(item)) + if (count < freq) freq = count + i += 1 + } + makeApprox(freq) + } + + /** + * Let X be a CMS, and let count_X[j, k] denote the value in X's 2-dimensional count table at row j and + * column k. Then the Count-Min sketch estimate of the inner product between A and B is the minimum inner + * product between their rows: estimatedInnerProduct = min_j (\sum_k count_A[j, k] * count_B[j, k]|) + */ + override def innerProduct(other: CMS[K]): Approximate[Long] = + other match { + case other: CMSInstance[?] => + require(other.depth == depth && other.width == width, "Tables must have the same dimensions.") + + def innerProductAtDepth(d: Int) = + (0 to (width - 1)).iterator.map { w => + countsTable.getCount((d, w)) * other.countsTable.getCount((d, w)) + }.sum + + val est = (0 to (depth - 1)).iterator.map(innerProductAtDepth).min + val minimum = + math.max(est - (eps * totalCount * other.totalCount).toLong, 0) + Approximate(minimum, est, est, 1 - delta) + case _ => other.innerProduct(this) + } + + override def +(item: K, count: Long): CMSInstance[K] = { + require(count >= 0, "count must be >= 0 (negative counts not implemented") + if (count != 0L) { + val newCountsTable = + (0 to (depth - 1)).foldLeft(countsTable) { case (table, row) => + val pos = (row, params.hashes(row)(item)) + table + (pos, count) + } + CMSInstance[K](newCountsTable, totalCount + count, params) + } else this + } + +} + +object CMSInstance { + + /** + * Initializes a [[CMSInstance]] with all zeroes, i.e. nothing has been counted yet. + */ + def apply[K](params: CMSParams[K]): CMSInstance[K] = { + val countsTable = CountsTable[K](CMSFunctions.depth(params.delta), CMSFunctions.width(params.eps)) + CMSInstance[K](countsTable, 0, params) + } + + /** + * The 2-dimensional table of counters used in the Count-Min sketch. Each row corresponds to a particular + * hash function. + */ + // TODO: implement a dense matrix type, and use it here + case class CountsTable[K](counts: Vector[Vector[Long]]) { + require(depth > 0, "Table must have at least 1 row.") + require(width > 0, "Table must have at least 1 column.") + + def depth: Int = counts.size + + def width: Int = counts(0).size + + def getCount(pos: (Int, Int)): Long = { + val (row, col) = pos + require(row < depth && col < width, "Position must be within the bounds of this table.") + counts(row)(col) + } + + /** + * Updates the count of a single cell in the table. + */ + def +(pos: (Int, Int), count: Long): CountsTable[K] = { + val (row, col) = pos + val currCount = getCount(pos) + val newCounts = + counts.updated(row, counts(row).updated(col, currCount + count)) + CountsTable[K](newCounts) + } + + /** + * Adds another counts table to this one, through element-wise addition. + */ + def ++(other: CountsTable[K]): CountsTable[K] = { + require(depth == other.depth && width == other.width, "Tables must have the same dimensions.") + val xss = this.counts.iterator + val yss = other.counts.iterator + val rows = Vector.newBuilder[Vector[Long]] + while (xss.hasNext) { + val xs = xss.next().iterator + val ys = yss.next().iterator + val row = Vector.newBuilder[Long] + while (xs.hasNext) row += (xs.next() + ys.next()) + rows += row.result() + } + CountsTable[K](rows.result()) + } + } + + object CountsTable { + + /** + * Creates a new [[CountsTable]] with counts initialized to all zeroes. + */ + def apply[K](depth: Int, width: Int): CountsTable[K] = + CountsTable[K](Vector.fill[Long](depth, width)(0L)) + + } + +} + +case class TopCMSParams[K](logic: HeavyHittersLogic[K]) + +/** + * A Count-Min sketch data structure that allows for (a) counting and frequency estimation of elements in a + * data stream and (b) tracking the heavy hitters among these elements. + * + * The logic of how heavy hitters are computed is pluggable, see [[HeavyHittersLogic]]. + * + * Tip: If you do not need to track heavy hitters, take a look at [[CMS]], which is more efficient in this + * case. + * + * =Usage= + * + * This example demonstrates how to count `Long` elements with [[TopCMS]], i.e. `K=Long`. + * + * Note that the actual counting is always performed with a `Long`, regardless of your choice of `K`. That is, + * the counting table behind the scenes is backed by `Long` values (at least in the current implementation), + * and thus the returned frequency estimates are always instances of `Approximate[Long]`. + * + * @example + * {{{ // Creates a monoid for a CMS that can count `Long` elements. val topPctCMSMonoid: + * TopPctCMSMonoid[Long] = { val eps = 0.001 val delta = 1E-10 val seed = 1 val heavyHittersPct = 0.1 + * TopPctCMS.monoid[Long](eps, delta, seed, heavyHittersPct) } + * + * // Creates a TopCMS instance that has counted the element `1L`. val topCMS: TopCMS[Long] = + * topPctCMSMonoid.create(1L) + * + * // Estimates the frequency of `1L` val estimate: Approximate[Long] = topCMS.frequency(1L) + * + * // What are the heavy hitters so far? val heavyHitters: Set[Long] = topCMS.heavyHitters }}} + * + * @tparam K + * The type used to identify the elements to be counted. + */ +sealed abstract class TopCMS[K](val cms: CMS[K], params: TopCMSParams[K]) + extends java.io.Serializable + with CMSCounting[K, TopCMS] + with CMSHeavyHitters[K] { + + override val eps: Double = cms.eps + + override val delta: Double = cms.delta + + override val totalCount: Long = cms.totalCount + + override val maxExactCountOpt: Option[Int] = cms.maxExactCountOpt + + override def frequency(item: K): Approximate[Long] = cms.frequency(item) + + override def innerProduct(other: TopCMS[K]): Approximate[Long] = + cms.innerProduct(other.cms) + + override def f2: Approximate[Long] = innerProduct(this) + + /** + * The pluggable logic with which heavy hitters are being tracked. + */ + override def heavyHittersLogic: HeavyHittersLogic[K] = params.logic + +} + +/** + * Zero element. Used for initialization. + */ +case class TopCMSZero[K](override val cms: CMS[K], params: TopCMSParams[K]) extends TopCMS[K](cms, params) { + + override val heavyHitters: Set[K] = Set.empty[K] + + override def +(item: K, count: Long): TopCMS[K] = + TopCMSInstance(cms, params) + (item, count) + + override def ++(other: TopCMS[K]): TopCMS[K] = other + +} + +/** + * Used for holding a single element, to avoid repeatedly adding elements from sparse counts tables. + */ +case class TopCMSItem[K](item: K, override val cms: CMS[K], params: TopCMSParams[K]) + extends TopCMS[K](cms, params) { + + override val heavyHitters: Set[K] = Set(item) + + override def +(x: K, count: Long): TopCMS[K] = toCMSInstance + (x, count) + + override def ++(other: TopCMS[K]): TopCMS[K] = other match { + case _: TopCMSZero[?] => this + case other: TopCMSItem[K] => toCMSInstance + other.item + case other: TopCMSInstance[K] => other + item + } + + private def toCMSInstance: TopCMSInstance[K] = { + val hhs = HeavyHitters.from(HeavyHitter(item, 1L)) + TopCMSInstance(cms, hhs, params) + } + +} + +object TopCMSInstance { + + def apply[K](cms: CMS[K], params: TopCMSParams[K]): TopCMSInstance[K] = + TopCMSInstance[K](cms, HeavyHitters.empty[K], params) + +} + +case class TopCMSInstance[K](override val cms: CMS[K], hhs: HeavyHitters[K], params: TopCMSParams[K]) + extends TopCMS[K](cms, params) { + + override def heavyHitters: Set[K] = hhs.items + + override def +(item: K, count: Long): TopCMSInstance[K] = { + require(count >= 0, "count must be >= 0 (negative counts not implemented") + if (count != 0L) { + val newCms = cms + (item, count) + val newHhs = + heavyHittersLogic.updateHeavyHitters(cms, newCms)(hhs, item, count) + TopCMSInstance[K](newCms, newHhs, params) + } else this + } + + override def ++(other: TopCMS[K]): TopCMS[K] = other match { + case _: TopCMSZero[?] => this + case other: TopCMSItem[K] => this + other.item + case other: TopCMSInstance[K] => + val newCms = cms ++ other.cms + val newHhs = heavyHittersLogic.updateHeavyHitters(newCms)(hhs, other.hhs) + TopCMSInstance(newCms, newHhs, params) + } + +} + +class TopCMSMonoid[K](emptyCms: CMS[K], logic: HeavyHittersLogic[K]) extends Monoid[TopCMS[K]] { + + val params: TopCMSParams[K] = TopCMSParams(logic) + + override val zero: TopCMS[K] = TopCMSZero[K](emptyCms, params) + + /** + * Combines the two sketches. + * + * The sketches must use the same hash functions. + */ + override def plus(left: TopCMS[K], right: TopCMS[K]): TopCMS[K] = { + require( + left.cms.params.hashes == right.cms.params.hashes, + "The sketches must use the same hash functions." + ) + left ++ right + } + + /** + * Creates a sketch out of a single item. + */ + def create(item: K): TopCMS[K] = + TopCMSItem[K](item, emptyCms + item, params) + + /** + * Creates a sketch out of multiple items. + */ + def create(data: Seq[K]): TopCMS[K] = + data.foldLeft(zero) { case (acc, x) => plus(acc, create(x)) } + + override def sum(sketches: TraversableOnce[TopCMS[K]]): TopCMS[K] = { + val topCandidates = scala.collection.mutable.Set.empty[K] + val summation = new CMSSummation(emptyCms.params) + sketches.iterator.foreach { sketch => + summation.updateInto(sketch.cms) + topCandidates ++= sketch.heavyHitters + } + val cms = summation.result + val ests = + topCandidates.map(k => HeavyHitter(k, cms.frequency(k).estimate)).toSet + val hhs = logic.purgeHeavyHitters(cms)(HeavyHitters(ests)) + TopCMSInstance(cms, hhs, params) + } + + override def sumOption(sketches: TraversableOnce[TopCMS[K]]): Option[TopCMS[K]] = + if (sketches.iterator.isEmpty) None else Some(sum(sketches)) +} + +class TopCMSAggregator[K](cmsMonoid: TopCMSMonoid[K]) extends MonoidAggregator[K, TopCMS[K], TopCMS[K]] { + + override def monoid: TopCMSMonoid[K] = cmsMonoid + + override def prepare(value: K): TopCMS[K] = monoid.create(value) + + override def present(cms: TopCMS[K]): TopCMS[K] = cms + +} + +/** + * Controls how a CMS that implements [[CMSHeavyHitters]] tracks heavy hitters. + */ +abstract class HeavyHittersLogic[K] extends java.io.Serializable { + + def updateHeavyHitters( + oldCms: CMS[K], + newCms: CMS[K] + )(hhs: HeavyHitters[K], item: K, count: Long): HeavyHitters[K] = { + val oldItemCount = oldCms.frequency(item).estimate + val oldHh = HeavyHitter[K](item, oldItemCount) + val newItemCount = oldItemCount + count + val newHh = HeavyHitter[K](item, newItemCount) + purgeHeavyHitters(newCms)(hhs - oldHh + newHh) + } + + def updateHeavyHitters(cms: CMS[K])(left: HeavyHitters[K], right: HeavyHitters[K]): HeavyHitters[K] = { + val candidates = (left.items ++ right.items).map { case i => + HeavyHitter[K](i, cms.frequency(i).estimate) + } + val newHhs = HeavyHitters.from(candidates) + purgeHeavyHitters(cms)(newHhs) + } + + def purgeHeavyHitters(cms: CMS[K])(hhs: HeavyHitters[K]): HeavyHitters[K] + +} + +/** + * Finds all heavy hitters, i.e., elements in the stream that appear at least `(heavyHittersPct * totalCount)` + * times. + * + * Every item that appears at least `(heavyHittersPct * totalCount)` times is output, and with probability `p + * >= 1 - delta`, no item whose count is less than `(heavyHittersPct - eps) * totalCount` is output. + * + * This also means that this parameter is an upper bound on the number of heavy hitters that will be tracked: + * the set of heavy hitters contains at most `1 / heavyHittersPct` elements. For example, if + * `heavyHittersPct=0.01` (or 0.25), then at most `1 / 0.01 = 100` items (or `1 / 0.25 = 4` items) will be + * tracked/returned as heavy hitters. This parameter can thus control the memory footprint required for + * tracking heavy hitters. + */ +case class TopPctLogic[K](heavyHittersPct: Double) extends HeavyHittersLogic[K] { + + require(0 < heavyHittersPct && heavyHittersPct < 1, "heavyHittersPct must lie in (0, 1)") + + override def purgeHeavyHitters(cms: CMS[K])(hitters: HeavyHitters[K]): HeavyHitters[K] = { + val minCount = heavyHittersPct * cms.totalCount + HeavyHitters[K](hitters.hhs.filter(_.count >= minCount)) + } + +} + +/** + * Tracks the top N heavy hitters, where `N` is defined by `heavyHittersN`. + * + * '''Warning:''' top-N computations are not associative. The effect is that a top-N CMS has an ordering bias + * (with regard to heavy hitters) when merging instances. This means merging heavy hitters across CMS + * instances may lead to incorrect, biased results: the outcome is biased by the order in which CMS instances + * / heavy hitters are being merged, with the rule of thumb being that the earlier a set of heavy hitters is + * being merged, the more likely is the end result biased towards these heavy hitters. + * + * @see + * Discussion in [[https://github.com/twitter/algebird/issues/353 Algebird issue 353]] + */ +case class TopNLogic[K](heavyHittersN: Int) extends HeavyHittersLogic[K] { + + require(heavyHittersN > 0, "heavyHittersN must be > 0") + + override def purgeHeavyHitters(cms: CMS[K])(hitters: HeavyHitters[K]): HeavyHitters[K] = { + val sorted = + hitters.hhs.toSeq.sortBy(hh => hh.count).takeRight(heavyHittersN) + HeavyHitters[K](sorted.toSet) + } + +} + +/** + * Containers for holding heavy hitter items and their associated counts. + */ +case class HeavyHitters[K](hhs: Set[HeavyHitter[K]]) extends java.io.Serializable { + + def -(hh: HeavyHitter[K]): HeavyHitters[K] = HeavyHitters[K](hhs - hh) + + def +(hh: HeavyHitter[K]): HeavyHitters[K] = HeavyHitters[K](hhs + hh) + + def ++(other: HeavyHitters[K]): HeavyHitters[K] = + HeavyHitters[K](hhs ++ other.hhs) + + def items: Set[K] = hhs.map(_.item) + +} + +object HeavyHitters { + + def empty[K]: HeavyHitters[K] = HeavyHitters(emptyHhs) + + private def emptyHhs[K]: Set[HeavyHitter[K]] = Set[HeavyHitter[K]]() + + def from[K](hhs: Set[HeavyHitter[K]]): HeavyHitters[K] = + hhs.foldLeft(empty[K])(_ + _) + + def from[K](hh: HeavyHitter[K]): HeavyHitters[K] = HeavyHitters(emptyHhs + hh) + +} + +case class HeavyHitter[K](item: K, count: Long) extends java.io.Serializable + +/** + * Monoid for Top-% based [[TopCMS]] sketches. + * + * =Usage= + * + * The type `K` is the type of items you want to count. You must provide an implicit `CMSHasher[K]` for `K`, + * and Algebird ships with several such implicits for commonly used types such as `Long` and `BigInt`. + * + * If your type `K` is not supported out of the box, you have two options: 1) You provide a "translation" + * function to convert items of your (unsupported) type `K` to a supported type such as Double, and then use + * the `contramap` function of [[CMSHasher]] to create the required `CMSHasher[K]` for your type (see the + * documentation of [[CMSHasher]] for an example); 2) You implement a `CMSHasher[K]` from scratch, using the + * existing CMSHasher implementations as a starting point. + * + * Note: Because Arrays in Scala/Java not have sane `equals` and `hashCode` implementations, you cannot safely + * use types such as `Array[Byte]`. Extra work is required for Arrays. For example, you may opt to convert + * `Array[T]` to a `Seq[T]` via `toSeq`, or you can provide appropriate wrapper classes. Algebird provides one + * such wrapper class, [[Bytes]], to safely wrap an `Array[Byte]` for use with CMS. + * + * @param cms + * A [[CMS]] instance, which is used for the counting and the frequency estimation performed by this class. + * @param heavyHittersPct + * A threshold for finding heavy hitters, i.e., elements that appear at least (heavyHittersPct * totalCount) + * times in the stream. + * @tparam K + * The type used to identify the elements to be counted. For example, if you want to count the occurrence of + * user names, you could map each username to a unique numeric ID expressed as a `Long`, and then count the + * occurrences of those `Long`s with a CMS of type `K=Long`. Note that this mapping between the elements of + * your problem domain and their identifiers used for counting via CMS should be bijective. We require a + * [[CMSHasher]] context bound for `K`, see [[CMSHasher]] for available implicits that can be imported. + * Which type K should you pick in practice? For domains that have less than `2^64` unique elements, you'd + * typically use `Long`. For larger domains you can try `BigInt`, for example. + */ +class TopPctCMSMonoid[K](cms: CMS[K], heavyHittersPct: Double = 0.01) + extends TopCMSMonoid[K](cms, TopPctLogic[K](heavyHittersPct)) + +object TopPctCMS { + + def monoid[K: CMSHasher]( + eps: Double, + delta: Double, + seed: Int, + heavyHittersPct: Double + ): TopPctCMSMonoid[K] = + new TopPctCMSMonoid[K](CMS(eps, delta, seed), heavyHittersPct) + + def monoid[K: CMSHasher](depth: Int, width: Int, seed: Int, heavyHittersPct: Double): TopPctCMSMonoid[K] = + monoid(CMSFunctions.eps(width), CMSFunctions.delta(depth), seed, heavyHittersPct) + + def aggregator[K: CMSHasher]( + eps: Double, + delta: Double, + seed: Int, + heavyHittersPct: Double + ): TopPctCMSAggregator[K] = + new TopPctCMSAggregator[K](monoid(eps, delta, seed, heavyHittersPct)) + + def aggregator[K: CMSHasher]( + depth: Int, + width: Int, + seed: Int, + heavyHittersPct: Double + ): TopPctCMSAggregator[K] = + aggregator(CMSFunctions.eps(width), CMSFunctions.delta(depth), seed, heavyHittersPct) + +} + +/** + * An Aggregator for [[TopPctCMS]]. Can be created using [[TopPctCMS.aggregator]]. + */ +case class TopPctCMSAggregator[K](cmsMonoid: TopPctCMSMonoid[K]) extends TopCMSAggregator(cmsMonoid) + +/** + * Monoid for top-N based [[TopCMS]] sketches. '''Use with care! (see warning below)''' + * + * =Warning: Adding top-N CMS instances (`++`) is an unsafe operation= + * + * Top-N computations are not associative. The effect is that a top-N CMS has an ordering bias (with regard to + * heavy hitters) when ''merging'' CMS instances (e.g. via `++`). This means merging heavy hitters across CMS + * instances may lead to incorrect, biased results: the outcome is biased by the order in which CMS instances + * / heavy hitters are being merged, with the rule of thumb being that the earlier a set of heavy hitters is + * being merged, the more likely is the end result biased towards these heavy hitters. + * + * The warning above only applies when ''adding CMS instances'' (think: `cms1 ++ cms2`). In comparison, heavy + * hitters are correctly computed when: + * + * - a top-N CMS instance is created from a single data stream, i.e. `Seq[K]` + * - items are added/counted individually, i.e. `cms + item` or `cms + (item, count)`. + * + * See the discussion in [[https://github.com/twitter/algebird/issues/353 Algebird issue 353]] for further + * details. + * + * =Alternatives= + * + * The following, alternative data structures may be better picks than a top-N based CMS given the warning + * above: + * + * - [[TopPctCMS]]: Has safe merge semantics for its instances including heavy hitters. + * - [[SpaceSaver]]: Has the same ordering bias than a top-N CMS, but at least it provides bounds on the + * bias. + * + * =Usage= + * + * The type `K` is the type of items you want to count. You must provide an implicit `CMSHasher[K]` for `K`, + * and Algebird ships with several such implicits for commonly used types such as `Long` and `BigInt`. + * + * If your type `K` is not supported out of the box, you have two options: 1) You provide a "translation" + * function to convert items of your (unsupported) type `K` to a supported type such as [[Double]], and then + * use the `contramap` function of [[CMSHasher]] to create the required `CMSHasher[K]` for your type (see the + * documentation of [[CMSHasher]] for an example); 2) You implement a `CMSHasher[K]` from scratch, using the + * existing CMSHasher implementations as a starting point. + * + * Note: Because Arrays in Scala/Java not have sane `equals` and `hashCode` implementations, you cannot safely + * use types such as `Array[Byte]`. Extra work is required for Arrays. For example, you may opt to convert + * `Array[T]` to a `Seq[T]` via `toSeq`, or you can provide appropriate wrapper classes. Algebird provides one + * such wrapper class, [[Bytes]], to safely wrap an `Array[Byte]` for use with CMS. + * + * @param cms + * A [[CMS]] instance, which is used for the counting and the frequency estimation performed by this class. + * @param heavyHittersN + * The maximum number of heavy hitters to track. + * @tparam K + * The type used to identify the elements to be counted. For example, if you want to count the occurrence of + * user names, you could map each username to a unique numeric ID expressed as a `Long`, and then count the + * occurrences of those `Long`s with a CMS of type `K=Long`. Note that this mapping between the elements of + * your problem domain and their identifiers used for counting via CMS should be bijective. We require a + * [[CMSHasher]] context bound for `K`, see [[CMSHasher]] for available implicits that can be imported. + * Which type K should you pick in practice? For domains that have less than `2^64` unique elements, you'd + * typically use `Long`. For larger domains you can try `BigInt`, for example. + */ +class TopNCMSMonoid[K](cms: CMS[K], heavyHittersN: Int = 100) + extends TopCMSMonoid[K](cms, TopNLogic[K](heavyHittersN)) + +object TopNCMS { + + def monoid[K: CMSHasher](eps: Double, delta: Double, seed: Int, heavyHittersN: Int): TopNCMSMonoid[K] = + new TopNCMSMonoid[K](CMS(eps, delta, seed), heavyHittersN) + + def monoid[K: CMSHasher](depth: Int, width: Int, seed: Int, heavyHittersN: Int): TopNCMSMonoid[K] = + monoid(CMSFunctions.eps(width), CMSFunctions.delta(depth), seed, heavyHittersN) + + def aggregator[K: CMSHasher]( + eps: Double, + delta: Double, + seed: Int, + heavyHittersN: Int + ): TopNCMSAggregator[K] = + new TopNCMSAggregator[K](monoid(eps, delta, seed, heavyHittersN)) + + def aggregator[K: CMSHasher](depth: Int, width: Int, seed: Int, heavyHittersN: Int): TopNCMSAggregator[K] = + aggregator(CMSFunctions.eps(width), CMSFunctions.delta(depth), seed, heavyHittersN) + +} + +/** + * An Aggregator for [[TopNCMS]]. Can be created using [[TopNCMS.aggregator]]. + */ +case class TopNCMSAggregator[K](cmsMonoid: TopNCMSMonoid[K]) extends TopCMSAggregator(cmsMonoid) + +/** + * K1 defines a scope for the CMS. For each k1, keep the top heavyHittersN associated k2 values. + */ +case class ScopedTopNLogic[K1, K2](heavyHittersN: Int) extends HeavyHittersLogic[(K1, K2)] { + + require(heavyHittersN > 0, "heavyHittersN must be > 0") + + override def purgeHeavyHitters( + cms: CMS[(K1, K2)] + )(hitters: HeavyHitters[(K1, K2)]): HeavyHitters[(K1, K2)] = { + val grouped = hitters.hhs.groupBy(hh => hh.item._1) + val (underLimit, overLimit) = grouped.partition { + _._2.size <= heavyHittersN + } + val sorted = overLimit.transform { case (_, hhs) => + hhs.toSeq.sortBy(hh => hh.count) + } + val purged = sorted.transform { case (_, hhs) => + hhs.takeRight(heavyHittersN) + } + HeavyHitters[(K1, K2)](purged.values.flatten.toSet ++ underLimit.values.flatten.toSet) + } + +} + +/* + * Monoid for Top-N values per key in an associative [[TopCMS]]. + * + * Typical use case for this might be (Country, City) pairs. For a stream of such + * pairs, we might want to keep track of the most popular cities for each country. + * + * This can, of course, be achieved using a Map[Country, TopNCMS[City]], but this + * requires storing one CMS per distinct Country. + * + * Similarly, one could attempt to use a TopNCMS[(Country, City)], but less common + * countries may not make the cut if N is not "very large". + * + * ScopedTopNCMSMonoid[Country, City] will avoid having one Country drown others + * out, while still only using a single CMS. + * + * In general the eviction of K1 is not supported, and all distinct K1 values must + * be retained. Therefore it is important to only use this Monoid when the number + * of distinct K1 values is known to be reasonably bounded. + */ +class ScopedTopNCMSMonoid[K1, K2](cms: CMS[(K1, K2)], heavyHittersN: Int = 100) + extends TopCMSMonoid[(K1, K2)](cms, ScopedTopNLogic[K1, K2](heavyHittersN)) + +object ScopedTopNCMS { + + def scopedHasher[K1: CMSHasher, K2: CMSHasher]: CMSHasher[(K1, K2)] = new CMSHasher[(K1, K2)] { + private val k1Hasher = implicitly[CMSHasher[K1]] + private val k2Hasher = implicitly[CMSHasher[K2]] + + override def hash(a: Int, b: Int, width: Int)(x: (K1, K2)): Int = { + val (k1, k2) = x + val xs = Seq(k1Hasher.hash(a, b, width)(k1), k2Hasher.hash(a, b, width)(k2), a, b) + (scala.util.hashing.MurmurHash3.seqHash(xs) & Int.MaxValue) % width + } + } + + def monoid[K1: CMSHasher, K2: CMSHasher]( + eps: Double, + delta: Double, + seed: Int, + heavyHittersN: Int + ): ScopedTopNCMSMonoid[K1, K2] = + new ScopedTopNCMSMonoid[K1, K2](CMS(eps, delta, seed)(scopedHasher[K1, K2]), heavyHittersN) + + def monoid[K1: CMSHasher, K2: CMSHasher]( + depth: Int, + width: Int, + seed: Int, + heavyHittersN: Int + ): ScopedTopNCMSMonoid[K1, K2] = + monoid(CMSFunctions.eps(width), CMSFunctions.delta(depth), seed, heavyHittersN) + + def aggregator[K1: CMSHasher, K2: CMSHasher]( + eps: Double, + delta: Double, + seed: Int, + heavyHittersN: Int + ): TopCMSAggregator[(K1, K2)] = + new TopCMSAggregator(monoid(eps, delta, seed, heavyHittersN)) + + def aggregator[K1: CMSHasher, K2: CMSHasher]( + depth: Int, + width: Int, + seed: Int, + heavyHittersN: Int + ): TopCMSAggregator[(K1, K2)] = + aggregator(CMSFunctions.eps(width), CMSFunctions.delta(depth), seed, heavyHittersN) + +} + +case class CMSHash[K: CMSHasher](a: Int, b: Int, width: Int) extends java.io.Serializable { + + /** + * Returns `a * x + b (mod p) (mod width)`. + */ + def apply(x: K): Int = implicitly[CMSHasher[K]].hash(a, b, width)(x) + +} + +/** + * This formerly held the instances that moved to object CMSHasher + * + * These instances are slow, but here for compatibility with old serialized data. For new code, avoid these + * and instead use the implicits found in the CMSHasher companion object. + */ +object CMSHasherImplicits { + + implicit object CMSHasherBigInt extends CMSHasher[BigInt] { + override def hash(a: Int, b: Int, width: Int)(x: BigInt): Int = + CMSHasher.hashBytes(a, b, width)(x.toByteArray) + } + + implicit object CMSHasherString extends CMSHasher[String] { + override def hash(a: Int, b: Int, width: Int)(x: String): Int = + CMSHasher.hashBytes(a, b, width)(x.getBytes("UTF-8")) + } + + def cmsHasherShort: CMSHasher[Short] = CMSHasher.cmsHasherShort +} diff --git a/algebird-core/src/main/scala-2.12+/DecayedVector.scala b/algebird-core/src/main/scala-2.12+/DecayedVector.scala new file mode 100644 index 000000000..18e816fe4 --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/DecayedVector.scala @@ -0,0 +1,75 @@ +/* +Copyright 2012 Twitter, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.algebird + +/** + * Represents a container class together with time. Its monoid consists of exponentially scaling the older + * value and summing with the newer one. + */ +object DecayedVector extends CompatDecayedVector { + def buildWithHalflife[C[_]](vector: C[Double], time: Double, halfLife: Double): DecayedVector[C] = + DecayedVector(vector, time * scala.math.log(2.0) / halfLife) + + def monoidWithEpsilon[C[_]]( + eps: Double + )(implicit vs: VectorSpace[Double, C], metric: Metric[C[Double]]): Monoid[DecayedVector[C]] = + new Monoid[DecayedVector[C]] { + override val zero = DecayedVector(vs.group.zero, Double.NegativeInfinity) + override def plus(left: DecayedVector[C], right: DecayedVector[C]) = + if (left.scaledTime <= right.scaledTime) { + scaledPlus(right, left, eps) + } else { + scaledPlus(left, right, eps) + } + } + + def forMap[K](m: Map[K, Double], scaledTime: Double): DecayedVector[Map[K, _]] = + DecayedVector[Map[K, _]](m, scaledTime) + def forMapWithHalflife[K](m: Map[K, Double], time: Double, halfLife: Double): DecayedVector[Map[K, _]] = + forMap(m, time * scala.math.log(2.0) / halfLife) + + def mapMonoidWithEpsilon[K]( + eps: Double + )(implicit + vs: VectorSpace[Double, Map[K, _]], + metric: Metric[Map[K, Double]] + ): Monoid[DecayedVector[Map[K, _]]] = + monoidWithEpsilon[Map[K, _]](eps) + + implicit def mapMonoid[K](implicit + vs: VectorSpace[Double, Map[K, _]], + metric: Metric[Map[K, Double]] + ): Monoid[DecayedVector[Map[K, _]]] = + mapMonoidWithEpsilon(-1.0) + + def scaledPlus[C[_]](newVal: DecayedVector[C], oldVal: DecayedVector[C], eps: Double)(implicit + vs: VectorSpace[Double, C], + metric: Metric[C[Double]] + ): DecayedVector[C] = { + implicit val mon: Monoid[C[Double]] = vs.group + val expFactor = scala.math.exp(oldVal.scaledTime - newVal.scaledTime) + val newVector = + Monoid.plus(newVal.vector, vs.scale(expFactor, oldVal.vector)) + if (eps < 0.0 || Metric.norm(newVector) > eps) { + DecayedVector(newVector, newVal.scaledTime) + } else { + DecayedVector(mon.zero, Double.NegativeInfinity) + } + } +} + +case class DecayedVector[C[_]](vector: C[Double], scaledTime: Double) diff --git a/algebird-core/src/main/scala-2.12+/DecayingCMS.scala b/algebird-core/src/main/scala-2.12+/DecayingCMS.scala new file mode 100644 index 000000000..54809e2a8 --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/DecayingCMS.scala @@ -0,0 +1,650 @@ +package com.twitter.algebird + +import java.lang.Double.{compare => cmp} +import java.lang.Math +import java.util.Arrays.deepHashCode +import scala.concurrent.duration.Duration +import scala.util.Random + +/** + * DecayingCMS is a module to build count-min sketch instances whose counts decay exponentially. + * + * Similar to a Map[K, com.twitter.algebird.DecayedValue], each key is associated with a single count value + * that decays over time. Unlike a map, the decyaing CMS is an approximate count -- in exchange for the + * possibility of over-counting, we can bound its size in memory. + * + * The intended use case is for metrics or machine learning where exact values aren't needed. + * + * You can expect the keys with the biggest values to be fairly accurate but the very small values (rare keys + * or very old keys) to be lost in the noise. For both metrics and ML this should be fine: you can't learn too + * much from very rare values. + * + * We recommend depth of at least 5, and width of at least 100, but you should do some experiments to + * determine the smallest parameters that will work for your use case. + */ +final class DecayingCMS[K]( + seed: Long, + val halfLife: Duration, + val depth: Int, // number of hashing functions + val width: Int, // number of table cells per hashing function + hasher: CMSHasher[K] +) extends Serializable { module => + + override def toString: String = + s"DecayingCMS(seed=$seed, halfLife=$halfLife, depth=$depth, width=$width)" + + @inline private def getNextLogScale( + logScale: Double, + oldTimeInHL: Double, + nowInHL: Double + ): Double = + if (nowInHL == oldTimeInHL) logScale else logScale + (nowInHL - oldTimeInHL) * log2 + + @inline private def getScale(logScale: Double, oldTimeInHL: Double, nowInHL: Double): Double = { + val logScale1 = getNextLogScale(logScale, oldTimeInHL, nowInHL) + Math.exp(-logScale1) + } + + val empty: CMS = + new CMS(Array.fill(depth)(Vector.fill[Double](width)(0.0)), 0.0, Double.NegativeInfinity) + + /** + * Represents a decaying scalar value at a particular point in time. + * + * The value decays according to halfLife. Another way to think about DoubleAt is that it represents a + * particular decay curve (and in particular, a point along that curve). Two DoubleAt values may be + * equivalent if they are two points on the same curve. + * + * The `timeToZero` and `timeToUnit` methods can be used to "normalize" DoubleAt values. If two DoubleAt + * values do not produce the same (approximate) Double values from these methods, they represent different + * curves. + */ + class DoubleAt private[algebird] (val value: Double, val timeInHL: Double) extends Serializable { + lhs => + + // this is not public because it's not safe in general -- you need + // to run a function that is time-commutative. + private[algebird] def map(f: Double => Double): DoubleAt = + new DoubleAt(f(value), timeInHL) + + // this is not public because it's not safe in general -- you need + // to run a function that is time-commutative. + private[algebird] def map2(rhs: DoubleAt)(f: (Double, Double) => Double): DoubleAt = + if (lhs.timeInHL < rhs.timeInHL) { + val x = lhs.scaledAt(rhs.timeInHL) + new DoubleAt(f(x, rhs.value), rhs.timeInHL) + } else if (lhs.timeInHL == rhs.timeInHL) { + new DoubleAt(f(lhs.value, rhs.value), rhs.timeInHL) + } else { + val y = rhs.scaledAt(lhs.timeInHL) + new DoubleAt(f(lhs.value, y), lhs.timeInHL) + } + + def unary_- : DoubleAt = new DoubleAt(-value, timeInHL) + def abs: DoubleAt = new DoubleAt(Math.abs(value), timeInHL) + def *(n: Double): DoubleAt = new DoubleAt(value * n, timeInHL) + + def +(rhs: DoubleAt): DoubleAt = map2(rhs)(_ + _) + def -(rhs: DoubleAt): DoubleAt = map2(rhs)(_ - _) + def min(rhs: DoubleAt): DoubleAt = map2(rhs)(Math.min) + def max(rhs: DoubleAt): DoubleAt = map2(rhs)(Math.max) + + def /(rhs: DoubleAt): Double = map2(rhs)(_ / _).value + + /** + * We consider two DoubleAt values equal not just if their elements are equal, but also if they represent + * the same value at different points of decay. + */ + def compare(rhs: DoubleAt): Int = { + val vc = cmp(lhs.value, rhs.value) + val tc = cmp(lhs.timeInHL, rhs.timeInHL) + if (vc == tc) vc + else if (tc == 0) vc + else if (vc == 0) tc + else if (tc < 0) cmp(lhs.scaledAt(rhs.timeInHL), rhs.value) + else cmp(lhs.value, rhs.scaledAt(lhs.timeInHL)) + } + + /** + * Time when this value will reach the smallest double value bigger than zero, unless we are already at + * zero in which case we return the current time + */ + def timeToZero: Double = + if (java.lang.Double.isNaN(value)) Double.NaN + else if (java.lang.Double.isInfinite(value)) Double.PositiveInfinity + else if (value == 0.0) timeInHL + else timeToUnit + DoubleAt.TimeFromUnitToZero + + /** + * This is the scaled time when the current value will reach 1 (or -1 for negative values) + * + * This method is a way of collapsing a DoubleAt into a single value (the time in the past or future where + * its value would be 1, the unit value). + */ + def timeToUnit: Double = + if (java.lang.Double.isNaN(value)) Double.NaN + else if (java.lang.Double.isInfinite(value)) Double.PositiveInfinity + else if (value == 0.0) Double.NegativeInfinity + else { + // solve for result: + // + // 1 = value * module.getScale(0.0, timeInHL, result) + // 1 = value * Math.exp(-getNextLogScale(0.0, timeInHL, result)) + // 1 / value = Math.exp(-getNextLogScale(0.0, timeInHL, result)) + // log(1 / value) = -getNextLogScale(0.0, timeInHL, result) + // -log(1 / value) = getNextLogScale(0.0, timeInHL, result) + // log(value) = getNextLogScale(0.0, timeInHL, result) + // log(value) = if (result == timeInHL) 0 else 0 + (result - timeInHL) * log2 + // log(value) = if (result == timeInHL) 0 else (result - timeInHL) * log2 + // + // log(value) = (result - timeInHL) * log2 + // log(value) / log2 = result - timeInHL + // log(value) / log2 + timeInHL = result + Math.log(Math.abs(value)) / log2 + timeInHL + } + + override def equals(that: Any): Boolean = + that match { + case d: DoubleAt => compare(d) == 0 + case _ => false + } + + override def hashCode: Int = + timeToUnit.## + + override def toString: String = + s"DoubleAt($value, $timeInHL)" + + def <(rhs: DoubleAt): Boolean = (lhs.compare(rhs)) < 0 + def <=(rhs: DoubleAt): Boolean = (lhs.compare(rhs)) <= 0 + def >(rhs: DoubleAt): Boolean = (lhs.compare(rhs)) > 0 + def >=(rhs: DoubleAt): Boolean = (lhs.compare(rhs)) >= 0 + + def time: Long = + toTimestamp(timeInHL) + + private def scaledAt(t: Double): Double = + if (value == 0.0) 0.0 + else value * module.getScale(0.0, timeInHL, t) + + def at(time: Long): Double = + if (value == 0.0) 0.0 + else value * module.getScale(0.0, timeInHL, fromTimestamp(time)) + } + + object DoubleAt { + def apply(x: Double, t: Long): DoubleAt = + new DoubleAt(x, fromTimestamp(t)) + + val zero: DoubleAt = + new DoubleAt(0.0, Double.NegativeInfinity) + + private val TimeFromUnitToZero: Double = + -Math.log(Double.MinPositiveValue) / log2 + } + + val totalCells: Int = depth * width + + val halfLifeSecs: Double = + halfLife.toMillis.toDouble / 1000.0 + + // TODO: consider a smaller number? + // we are trading accuracy for possible performence + private[this] val maxLogScale: Double = 20.0 + + /** + * Allocate an empty array of row. + * + * The elements start as null. It's an important optimization _not_ to allocate vectors here, since we're + * often building up cells mutably. + */ + private def allocCells(): Array[Vector[Double]] = + new Array[Vector[Double]](depth) + + def toTimestamp(t: Double): Long = + (t * halfLifeSecs * 1000.0).toLong + + def fromTimestamp(t: Long): Double = + (t.toDouble / 1000.0) / halfLifeSecs + + val hashFns: Array[K => Int] = { + val rng = new Random(seed) + def genPos(): Int = + rng.nextInt() match { + case 0 => genPos() + case n => n & 0x7fffffff + } + + (0 until depth).map { _ => + val n = genPos() + (k: K) => hasher.hash(n, 0, width)(k) + }.toArray + } + + private final val log2 = Math.log(2.0) + + /** + * The idealized formula for the updating current value for a key (y0 -> y1) is given as: + * + * delta = (t1 - t0) / halflife y1 = y0 * 2^(-delta) + n + * + * However, we want to avoid having to rescale every single cell every time we update; i.e. a cell with a + * zero value should continue to have a zero value when n=0. + * + * Therefore, we introduce a change of variable to cell values (z) along with a scale factor (scale), and + * the following formula: + * + * (1) zN = yN * scaleN + * + * Our constraint is expressed as: + * + * (2) If n=0, z1 = z0 + * + * In that case: + * + * (3) If n=0, (y1 * scale1) = (y0 * scale0) (4) Substituting for y1, (y0 * 2^(-delta) + 0) * scale1 = y0 * + * scale0 (5) 2^(-delta) * scale1 = scale0 (6) scale1 = scale0 * 2^(delta) + * + * Also, to express z1 in terms of z0, we say: + * + * (7) z1 = y1 * scale1 (8) z1 = (y0 * 2^(-delta) + n) * scale1 (9) z1 = ((z0 / scale0) * 2^(-delta) + n) * + * scale1 (10) z1 / scale1 = (z0 / (scale1 * 2^(-delta))) * 2^(-delta) + n (11) z1 / scale1 = z0 / scale1 + + * n (12) z1 = z0 + n * scale1 + * + * So, for cells where n=0, we just update scale0 to scale1, and for cells where n is non-zero, we update z1 + * in terms of z0 and scale1. + * + * If we convert scale to logscale, we have: + * + * (13) logscale1 = logscale0 + delta * log(2) (14) z1 = z0 + n * exp(logscale1) + * + * When logscale1 gets big, we start to distort z1. For example, exp(36) is close to 2^53. We can measure + * when n * exp(logscale1) gets big, and in those cases we can rescale all our cells (set each z to its + * corresponding y) and set the logscale to 0. + * + * (15) y1 = z1 / scale1 (16) y1 = z1 / exp(logscale1) (17) y1 = z1 * exp(-logscale1) + */ + final class CMS( + val cells: Array[Vector[Double]], + val logScale: Double, + val timeInHL: Double + ) extends Serializable { + + @inline private def scale: Double = + Math.exp(-logScale) + + override def toString: String = { + val s = cells.iterator.map(_.toString).mkString("Array(", ", ", ")") + s"CMS($s, $logScale, $timeInHL)" + } + + override def hashCode: Int = + deepHashCode(cells.asInstanceOf[Array[Object]]) * 59 + + logScale.## * 17 + + timeInHL.## * 37 + + 19 + + // unfortunately we can't check the path-dependent type of this + // CMS, which we signal by using a type projection here. + override def equals(any: Any): Boolean = + any match { + case that: DecayingCMS[?]#CMS => + this.logScale == that.logScale && + this.timeInHL == that.timeInHL && + this.cells.length == that.cells.length && { + var i = 0 + while (i < depth) { + if (this.cells(i) != that.cells(i)) return false + i += 1 + } + true + } + case _ => + false + } + + def lastUpdateTime: Long = + toTimestamp(timeInHL) + + /** + * Provide lower and upper bounds on values returned for any possible key. + * + * The first value is a lower bound: even keys that have never been counted will return this value or + * greater. This will be zero unless the CMS is saturated. + * + * The second value is an upper bound: the key with the largest cardinality will not be reported as being + * larger than this value (though it might be reported as being smaller). + * + * Together these values indicate how saturated and skewed the CMS might be. + */ + def range: (DoubleAt, DoubleAt) = { + var minMinimum = Double.PositiveInfinity + var minMaximum = Double.PositiveInfinity + var i = 0 + while (i < cells.length) { + val it = cells(i).iterator + var localMax = it.next() // we know it doesn't start empty + if (localMax < minMinimum) minMinimum = localMax + while (it.hasNext) { + val n = it.next() + if (n > localMax) localMax = n + else if (n < minMinimum) minMinimum = n + } + if (localMax < minMaximum) minMaximum = localMax + i += 1 + } + + val s = scale + def sc(x: Double): DoubleAt = + new DoubleAt(if (x == 0.0) 0.0 else x * s, timeInHL) + + (sc(minMinimum), sc(minMaximum)) + } + + /** + * Returns the square-root of the inner product of two decaying CMSs. + * + * We want the result to decay at the same rate as the CMS for this method to be valid. Taking the square + * root ensures that this is true. Without it, we would violate the following equality (assuming we had + * at() on a CMS): + * + * x.innerProduct(y).at(t) = x.at(t).innerProduct(y.at(t)) + * + * This is why we don't support innerProduct, only innerProductRoot. + */ + def innerProductRoot(that: CMS): DoubleAt = { + var i = 0 + var res = Double.PositiveInfinity + val t = Math.max(this.timeInHL, that.timeInHL) + val scale = this.getScale(t) * that.getScale(t) + while (i < depth) { + var sum = 0.0 + val it0 = this.cells(i).iterator + val it1 = that.cells(i).iterator + while (it0.hasNext) { + val x = it0.next() * it1.next() + if (x != 0.0) sum += x + } + if (sum < res) res = sum + i += 1 + } + val x = if (res != 0.0) Math.sqrt(res * scale) else 0.0 + new DoubleAt(x, t) + } + + def l2Norm: DoubleAt = + innerProductRoot(this) + + def scale(x: Double): CMS = + if (java.lang.Double.isNaN(x)) { + throw new IllegalArgumentException(s"invalid scale: $x") + } else if (x < 0.0) { + throw new IllegalArgumentException(s"negative scale is not allowed: $x") + } else if (x == 0.0) { + module.empty + } else { + val s = logScale + Math.log(x) + val c = new CMS(cells, s, timeInHL) + if (s > maxLogScale) c.rescaleTo(timeInHL) else c + } + + /** + * Get the total count of all items in the CMS. + * + * The total is the same as the l1Norm, since we don't allow negative values. + * + * Total is one of the few non-approximate statistics that DecayingCMS supports. We expect the total to be + * exact (except for floating-point error). + */ + def total: DoubleAt = { + val n = cells(0).sum + val x = if (n == 0.0) 0.0 else scale * n + new DoubleAt(x, timeInHL) + } + + def get(k: K): DoubleAt = { + var minValue = Double.PositiveInfinity + var didx = 0 + while (didx < depth) { + val i = hashFns(didx)(k) + val inner = cells(didx) + val value = inner(i) + if (value < minValue) minValue = value + didx += 1 + } + val x = if (minValue == 0.0) 0.0 else scale * minValue + new DoubleAt(x, timeInHL) + } + + def getScale(t: Double): Double = + module.getScale(logScale, timeInHL, t) + + private final def nextLogScale(t: Double): Double = + module.getNextLogScale(logScale, timeInHL, t) + + def +(other: CMS): CMS = { + val x = this + val y = other + val timeInHL = Math.max(x.timeInHL, y.timeInHL) + val cms = new CMS(allocCells(), 0.0, timeInHL) + + val xscale = x.getScale(timeInHL) + val yscale = y.getScale(timeInHL) + + // a zero count is zero, no matter, how big the scale is. + @inline def prod(x: Double, y: Double): Double = + if (x == 0.0) 0.0 else x * y + + var i = 0 + while (i < depth) { + val left = x.cells(i) + val right = y.cells(i) + var j = 0 + val bldr = rowBuilder() + while (j < width) { + bldr += prod(left(j), xscale) + prod(right(j), yscale) + j += 1 + } + cms.cells(i) = bldr.result() + i += 1 + } + cms + } + + def add(t: Long, k: K, n: Double): CMS = + scaledAdd(fromTimestamp(t), k, n) + + // TODO: we could allocate a mutable scratch pad, write all the + // values into it, and then build a CMS out of it. if items is + // very small, this would be less efficient than what we're doing + // now. probably the "ideal" solution would be determine how many + // items there are. if we have fewer than ~width items, this + // approach is fine. for more, a scratch pad would be better + // (assuming we wrote that code). + // + // alternately, you could map items into (zero + item) and then + // use the monoid's sum to boil it down. + // + // we only use this in testing currently so the current code is + // fine until we rely on it in production. any change here should + // probably include benchmarks justifying the design. + def bulkAdd(items: Iterable[(Long, K, Double)]): CMS = + items.foldLeft(this) { case (c, (t, k, v)) => c.add(t, k, v) } + + private[algebird] def scaledAdd(ts1: Double, k: K, n: Double): CMS = + if (n < 0.0) { + val t = toTimestamp(ts1) + throw new IllegalArgumentException( + s"we can only add non-negative numbers to a CMS, got $n for key: $k at time: $t" + ) + } else if (n == 0.0) { + this + } else { + val logScale1 = nextLogScale(ts1) + if (logScale1 > maxLogScale) { + rescaleTo(ts1).scaledAdd(ts1, k, n) + } else { + val increment = n * Math.exp(logScale1) + val cells1 = allocCells() + var didx = 0 + while (didx < depth) { + val cell = cells(didx) + val w = hashFns(didx)(k) + cells1(didx) = cell.updated(w, cell(w) + increment) + didx += 1 + } + new CMS(cells1, logScale1, ts1) + } + } + + // Set the scale back to 0.0 + // input time is in half-lives + private[algebird] def rescaleTo(ts: Double): CMS = { + val logScale1 = nextLogScale(ts) + val expL = Math.exp(-logScale1) + if (expL == 0.0) { + new CMS(monoid.zero.cells, 0.0, ts) + } else { + val cms = new CMS(allocCells(), 0.0, ts) + var i = 0 + while (i < depth) { + val ci = cells(i) + cms.cells(i) = ci.map(_ * expL) + i += 1 + } + cms + } + } + } + + private def rowBuilder() = { + val bldr = Vector.newBuilder[Double] + bldr.sizeHint(width) + bldr + } + + object CMS { + + implicit val monoidForCMS: Monoid[CMS] = + new Monoid[CMS] { + + def zero: CMS = module.empty + + def plus(x: CMS, y: CMS): CMS = + x + y + + /** + * Turn a flat array into an array of vectors. + */ + private def scratchToCells(scratch: Array[Double]): Array[Vector[Double]] = { + val cells = new Array[Vector[Double]](depth) + var i = 0 + while (i < depth) { + var j = i * width + val limit = j + width + val bldr = rowBuilder() + while (j < limit) { + bldr += scratch(j) + j += 1 + } + cells(i) = bldr.result() + i += 1 + } + cells + } + + /** + * This method sums the first `num` items in `arr`. + */ + private def innerSum(arr: Array[CMS], num: Int): CMS = + if (num == 0) zero + else if (num == 1) arr(0) + else if (num == 2) plus(arr(0), arr(1)) + else { + // start with zero + val scratch: Array[Double] = new Array(totalCells) + + val latestTimeInHL: Double = + arr.iterator.take(num).map(cms => cms.timeInHL).max + + var i = 0 + while (i < num) { + val cms = arr(i) + val scale = cms.getScale(latestTimeInHL) + var j = 0 + while (j < depth) { + val row = cms.cells(j) + val stride = j * width + var k = 0 + while (k < width) { + val n = row(k) + if (n > 0.0) { + scratch(stride + k) += scale * n + } + k += 1 + } + j += 1 + } + i += 1 + } + + val cells = scratchToCells(scratch) + + new CMS(cells, 0.0, latestTimeInHL) + } + + override def sumOption(xs: TraversableOnce[CMS]): Option[CMS] = { + + val it: Iterator[CMS] = xs.toIterator + val ChunkSize = 1000 + + // the idea here is that we read up to 1000 CMS values into + // a fixed array, crunch them down to a single CMS, store it + // in the first array index, read up to 999 more CMS values + // in, crunch them down, and so on. + var i = 0 + val arr = new Array[CMS](ChunkSize) + while (it.hasNext) { + while (it.hasNext && i < ChunkSize) { + arr(i) = it.next() + i += 1 + } + if (i > 1) { + arr(0) = innerSum(arr, i) + } + i = 1 + } + if (i == 0) None else Some(arr(0)) + } + } + } + + val monoid: Monoid[CMS] = CMS.monoidForCMS +} + +object DecayingCMS { + + /** + * Construct a DecayingCMS module. + * + * The seed is used to initialize the hash families used by the count-min sketch. Using the same seed will + * always produce the same hash family. + * + * Half-life determines the rate at which values in the CMS decay. If a key was counted once at time t, by + * time (t + halfLife), the value for that key will be 0.5. After enough half lives the value will decay to + * zero. + * + * The size of the CMS in bytes is O(depth * width). + * + * Width controls the relative error due to over-counting (approximately 1/width). For 1% error, use + * width=100, for 0.1% error, use width=1000, etc. + * + * Depth controls the probability the error bounds are broken and that probability scales with exp(-alpha * + * depth) so, a small depth (e.g. 5-10) is fine. Each update requires O(depth) work so you want to keep this + * as small as possible. + */ + def apply[K](seed: Long, halfLife: Duration, depth: Int, width: Int)(implicit + hasher: CMSHasher[K] + ): DecayingCMS[K] = + new DecayingCMS(seed, halfLife, depth, width, hasher) +} diff --git a/algebird-core/src/main/scala-2.12+/FoldCompat.scala b/algebird-core/src/main/scala-2.12+/FoldCompat.scala new file mode 100644 index 000000000..90bde5999 --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/FoldCompat.scala @@ -0,0 +1,26 @@ +package com.twitter.algebird + +private[algebird] trait FoldApplicativeCompat { + + /** + * "import Fold.applicative" will bring the Applicative instance into scope. See FoldApplicative. + */ + implicit def applicative[I]: Applicative[Fold[I, _]] = + new FoldApplicative[I] +} + +/** + * Folds are Applicatives! + */ +class FoldApplicative[I] extends Applicative[Fold[I, _]] { + override def map[T, U](mt: Fold[I, T])(fn: T => U): Fold[I, U] = + mt.map(fn) + override def apply[T](v: T): Fold[I, T] = + Fold.const(v) + override def join[T, U](mt: Fold[I, T], mu: Fold[I, U]): Fold[I, (T, U)] = + mt.join(mu) + override def sequence[T](ms: Seq[Fold[I, T]]): Fold[I, Seq[T]] = + Fold.sequence(ms) + override def joinWith[T, U, V](mt: Fold[I, T], mu: Fold[I, U])(fn: (T, U) => V): Fold[I, V] = + mt.joinWith(mu)(fn) +} diff --git a/algebird-core/src/main/scala-2.12+/Interval.scala b/algebird-core/src/main/scala-2.12+/Interval.scala new file mode 100644 index 000000000..6a1645d16 --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/Interval.scala @@ -0,0 +1,380 @@ +/* + Copyright 2013 Twitter, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +package com.twitter.algebird + +// TODO this is clearly more general than summingbird, and should be extended to be a ring (add union, etc...) + +/** + * Represents a single interval on a T with an Ordering + */ +sealed trait Interval[T] extends java.io.Serializable { + def contains(t: T)(implicit ord: Ordering[T]): Boolean + + def intersect(that: Interval[T])(implicit ord: Ordering[T]): Interval[T] + final def apply(t: T)(implicit ord: Ordering[T]): Boolean = contains(t) + final def &&(that: Interval[T])(implicit ord: Ordering[T]): Interval[T] = intersect(that) + + /** + * Map the Interval with a non-decreasing function. If you use a non-monotonic function (like x^2) then the + * result is meaningless. TODO: It might be good to have types for these properties in algebird. + */ + def mapNonDecreasing[U](fn: T => U): Interval[U] +} + +case class Universe[T]() extends Interval[T] { + override def contains(t: T)(implicit ord: Ordering[T]): Boolean = true + override def intersect(that: Interval[T])(implicit ord: Ordering[T]): Interval[T] = + that + override def mapNonDecreasing[U](fn: T => U): Interval[U] = Universe() +} + +case class Empty[T]() extends Interval[T] { + override def contains(t: T)(implicit ord: Ordering[T]): Boolean = false + override def intersect(that: Interval[T])(implicit ord: Ordering[T]): Interval[T] = + this + override def mapNonDecreasing[U](fn: T => U): Interval[U] = Empty() +} + +object Interval extends java.io.Serializable { + + /** + * Class that only exists so that [[leftClosedRightOpen]] and [[leftOpenRightClosed]] can retain the type + * information of the returned interval. The compiler doesn't know anything about ordering, so without + * [[MaybeEmpty]] the only valid return type is Interval[T]. + */ + sealed abstract class MaybeEmpty[T, NonEmpty[t] <: Interval[t]] { + def isEmpty: Boolean + } + object MaybeEmpty { + + /** + * Represents an empty interval. + */ + case class SoEmpty[T, NonEmpty[t] <: Interval[t]]() extends MaybeEmpty[T, NonEmpty] { + override def isEmpty: Boolean = true + } + + /** + * Represents a non-empty interval. + */ + case class NotSoEmpty[T, NonEmpty[t] <: Interval[t]](get: NonEmpty[T]) extends MaybeEmpty[T, NonEmpty] { + override def isEmpty: Boolean = false + } + } + + type GenIntersection[T] = Intersection[Lower, Upper, T] + type InLowExUp[T] = Intersection[InclusiveLower, ExclusiveUpper, T] + type InLowInUp[T] = Intersection[InclusiveLower, InclusiveUpper, T] + type ExLowExUp[T] = Intersection[ExclusiveLower, ExclusiveUpper, T] + type ExLowInUp[T] = Intersection[ExclusiveLower, InclusiveUpper, T] + + implicit def monoid[T: Ordering]: Monoid[Interval[T]] = + Monoid.from[Interval[T]](Universe[T]())(_ && _) + + // Automatically convert from a MaybeEmpty instance + implicit def fromMaybeEmpty[T, NonEmpty[t] <: Interval[t]](me: MaybeEmpty[T, NonEmpty]): Interval[T] = + me match { + case MaybeEmpty.SoEmpty() => Empty() + case MaybeEmpty.NotSoEmpty(i) => i + } + + def leftClosedRightOpen[T: Ordering](lower: T, upper: T): MaybeEmpty[T, InLowExUp] = + if (Ordering[T].lt(lower, upper)) + MaybeEmpty.NotSoEmpty[T, InLowExUp](Intersection(InclusiveLower(lower), ExclusiveUpper(upper))) + else MaybeEmpty.SoEmpty[T, InLowExUp]() + + def leftOpenRightClosed[T: Ordering](lower: T, upper: T): MaybeEmpty[T, ExLowInUp] = + if (Ordering[T].lt(lower, upper)) + MaybeEmpty.NotSoEmpty[T, ExLowInUp](Intersection(ExclusiveLower(lower), InclusiveUpper(upper))) + else MaybeEmpty.SoEmpty[T, ExLowInUp]() + + def closed[T: Ordering](lower: T, upper: T): MaybeEmpty[T, InLowInUp] = + if (Ordering[T].lteq(lower, upper)) + MaybeEmpty.NotSoEmpty[T, InLowInUp](Intersection(InclusiveLower(lower), InclusiveUpper(upper))) + else MaybeEmpty.SoEmpty[T, InLowInUp]() + + def open[T: Ordering](lower: T, upper: T): MaybeEmpty[T, ExLowExUp] = + if (Ordering[T].lt(lower, upper)) + MaybeEmpty.NotSoEmpty[T, ExLowExUp](Intersection(ExclusiveLower(lower), ExclusiveUpper(upper))) + else MaybeEmpty.SoEmpty[T, ExLowExUp]() + + /** + * This is here for binary compatibility reasons. These methods should be moved to Interval, which should + * also be an abstract class for better binary compatibility at the next incompatible change + */ + implicit final class IntervalMethods[T](val intr: Interval[T]) extends AnyVal { + def isEmpty(implicit succ: Successible[T], pred: Predecessible[T]): Boolean = intr match { + case Empty() => true + case Universe() => false + case Intersection(InclusiveLower(l), ExclusiveUpper(u)) => + !succ.ordering.lt(l, u) + case Intersection(InclusiveLower(l), InclusiveUpper(u)) => + !succ.ordering.lteq(l, u) + case Intersection(ExclusiveLower(l), ExclusiveUpper(u)) => + !succ.next(l).exists(succ.ordering.lt(_, u)) + case Intersection(ExclusiveLower(l), InclusiveUpper(u)) => + !succ.next(l).exists(succ.ordering.lteq(_, u)) + case InclusiveLower(_) => false // we at least have l + case InclusiveUpper(_) => false // false // we at least have u + case ExclusiveLower(l) => + succ.next(l).isEmpty + case ExclusiveUpper(u) => + pred.prev(u).isEmpty + } + + /** + * If this returns Some(t), then intr.contains(t) and there is no s less than t such that intr.contains(s) + * + * if this returns None, it may be Empty, Upper or Universe + */ + def boundedLeast(implicit succ: Successible[T]): Option[T] = intr match { + case Empty() => None + case Universe() => None + case _: Upper[?] => None + case i @ Intersection(_, _) => i.least + case l: Lower[?] => l.least + } + + /** + * If this returns Some(t), then intr.contains(t) and there is no s greater than t such that + * intr.contains(s) + * + * if this returns None, it may be Empty, Lower, or Universe + */ + def boundedGreatest(implicit pred: Predecessible[T]): Option[T] = + intr match { + case Empty() => None + case Universe() => None + case _: Lower[?] => None + case i @ Intersection(_, _) => i.greatest + case u: Upper[?] => u.greatest + } + } +} + +// Marker traits to keep lower on the left in Intersection +sealed trait Lower[T] extends Interval[T] { + + /** + * This may give a false positive (but should try not to). Note the case of (0,1) for the integers. If they + * were doubles, this would intersect, but since there are no members of the set Int that are bigger than 0 + * and less than 1, they don't really intersect. So, ordering is not enough here. You need a stronger + * notion, which we don't have a typeclass for. + */ + def intersects(u: Upper[T])(implicit ord: Ordering[T]): Boolean + + /** + * The smallest value that is contained here This is an Option, because of cases like + * ExclusiveLower(Int.MaxValue) which are pathological and equivalent to Empty + */ + def least(implicit s: Successible[T]): Option[T] + def strictLowerBound(implicit p: Predecessible[T]): Option[T] + + /** + * Iterates all the items in this Lower[T] from lowest to highest + */ + def toIterable(implicit s: Successible[T]): Iterable[T] = + least match { + case Some(l) => s.iterateNext(l) + case None => Iterable.empty + } +} +sealed trait Upper[T] extends Interval[T] { + + /** + * The smallest value that is contained here This is an Option, because of cases like + * ExclusiveUpper(Int.MinValue), which are pathological and equivalent to Empty + */ + def greatest(implicit p: Predecessible[T]): Option[T] + // The smallest value that is not present + def strictUpperBound(implicit s: Successible[T]): Option[T] + + /** + * Iterates all the items in this Upper[T] from highest to lowest + */ + def toIterable(implicit p: Predecessible[T]): Iterable[T] = + greatest match { + case Some(g) => p.iteratePrev(g) + case None => Iterable.empty + } +} + +case class InclusiveLower[T](lower: T) extends Interval[T] with Lower[T] { + override def contains(t: T)(implicit ordering: Ordering[T]): Boolean = + ordering.lteq(lower, t) + override def intersect(that: Interval[T])(implicit ordering: Ordering[T]): Interval[T] = that match { + case Universe() => this + case Empty() => that + case ub @ InclusiveUpper(_) => + if (intersects(ub)) Intersection(this, ub) else Empty() + case ub @ ExclusiveUpper(_) => + if (intersects(ub)) Intersection(this, ub) else Empty() + case InclusiveLower(thatlb) => + if (ordering.gt(lower, thatlb)) this else that + case ExclusiveLower(thatlb) => + if (ordering.gt(lower, thatlb)) this else that + case Intersection(thatL, thatU) => (this && thatL) && thatU + } + override def intersects(u: Upper[T])(implicit ordering: Ordering[T]): Boolean = + u match { + case InclusiveUpper(upper) => ordering.lteq(lower, upper) + case ExclusiveUpper(upper) => ordering.lt(lower, upper) + } + override def least(implicit s: Successible[T]): Option[T] = Some(lower) + override def strictLowerBound(implicit p: Predecessible[T]): Option[T] = p.prev(lower) + override def mapNonDecreasing[U](fn: T => U): Interval[U] = InclusiveLower(fn(lower)) +} +case class ExclusiveLower[T](lower: T) extends Interval[T] with Lower[T] { + override def contains(t: T)(implicit ordering: Ordering[T]): Boolean = + ordering.lt(lower, t) + override def intersect(that: Interval[T])(implicit ordering: Ordering[T]): Interval[T] = that match { + case Universe() => this + case Empty() => that + case ub @ InclusiveUpper(_) => + if (intersects(ub)) Intersection(this, ub) else Empty() + case ub @ ExclusiveUpper(_) => + if (intersects(ub)) Intersection(this, ub) else Empty() + case InclusiveLower(thatlb) => + if (ordering.gteq(lower, thatlb)) this else that + case ExclusiveLower(thatlb) => + if (ordering.gteq(lower, thatlb)) this else that + case Intersection(thatL, thatU) => (this && thatL) && thatU + } + override def intersects(u: Upper[T])(implicit ordering: Ordering[T]): Boolean = + u match { + case InclusiveUpper(upper) => ordering.lt(lower, upper) + case ExclusiveUpper(upper) => + ordering.lt(lower, upper) // This is a false positive for (x, next(x)) + } + override def least(implicit s: Successible[T]): Option[T] = s.next(lower) + override def strictLowerBound(implicit p: Predecessible[T]): Option[T] = Some(lower) + override def mapNonDecreasing[U](fn: T => U): Interval[U] = ExclusiveLower(fn(lower)) +} +case class InclusiveUpper[T](upper: T) extends Interval[T] with Upper[T] { + override def contains(t: T)(implicit ordering: Ordering[T]): Boolean = + ordering.lteq(t, upper) + override def greatest(implicit p: Predecessible[T]): Option[T] = Some(upper) + // The smallest value that is not present + override def strictUpperBound(implicit s: Successible[T]): Option[T] = s.next(upper) + override def intersect(that: Interval[T])(implicit ordering: Ordering[T]): Interval[T] = that match { + case Universe() => this + case Empty() => that + case lb @ InclusiveLower(_) => + if (lb.intersects(this)) Intersection(lb, this) else Empty() + case lb @ ExclusiveLower(_) => + if (lb.intersects(this)) Intersection(lb, this) else Empty() + case InclusiveUpper(thatub) => + if (ordering.lt(upper, thatub)) this else that + case ExclusiveUpper(thatub) => + if (ordering.lt(upper, thatub)) this else that + case Intersection(thatL, thatU) => thatL && (this && thatU) + } + override def mapNonDecreasing[U](fn: T => U): Interval[U] = InclusiveUpper(fn(upper)) +} +case class ExclusiveUpper[T](upper: T) extends Interval[T] with Upper[T] { + override def contains(t: T)(implicit ordering: Ordering[T]): Boolean = + ordering.lt(t, upper) + override def greatest(implicit p: Predecessible[T]): Option[T] = p.prev(upper) + // The smallest value that is not present + override def strictUpperBound(implicit s: Successible[T]): Option[T] = Some(upper) + override def intersect(that: Interval[T])(implicit ordering: Ordering[T]): Interval[T] = that match { + case Universe() => this + case Empty() => that + case lb @ InclusiveLower(_) => + if (lb.intersects(this)) Intersection(lb, this) else Empty() + case lb @ ExclusiveLower(_) => + if (lb.intersects(this)) Intersection(lb, this) else Empty() + case InclusiveUpper(thatub) => + if (ordering.lteq(upper, thatub)) this else that + case ExclusiveUpper(thatub) => + if (ordering.lteq(upper, thatub)) this else that + case Intersection(thatL, thatU) => thatL && (this && thatU) + } + override def mapNonDecreasing[U](fn: T => U): Interval[U] = ExclusiveUpper(fn(upper)) +} + +case class Intersection[L[t] <: Lower[t], U[t] <: Upper[t], T](lower: L[T], upper: U[T]) extends Interval[T] { + override def contains(t: T)(implicit ordering: Ordering[T]): Boolean = + lower.contains(t) && upper.contains(t) + override def intersect(that: Interval[T])(implicit ordering: Ordering[T]): Interval[T] = that match { + case Universe() => this + case Empty() => that + case lb @ InclusiveLower(_) => (lb && lower) && upper + case lb @ ExclusiveLower(_) => (lb && lower) && upper + case ub @ InclusiveUpper(_) => lower && (ub && upper) + case ub @ ExclusiveUpper(_) => lower && (ub && upper) + case Intersection(thatL, thatU) => (lower && thatL) && (upper && thatU) + } + override def mapNonDecreasing[T1](fn: T => T1): Interval[T1] = { + val newLower = lower match { + case InclusiveLower(l) => InclusiveLower(fn(l)) + case ExclusiveLower(l) => ExclusiveLower(fn(l)) + } + val newUpper = upper match { + case InclusiveUpper(u) => InclusiveUpper(fn(u)) + case ExclusiveUpper(u) => ExclusiveUpper(fn(u)) + } + Intersection(newLower, newUpper) + } + + def least(implicit s: Successible[T]): Option[T] = + lower.least.filter(upper.contains(_)(s.ordering)) + + /** + * Goes from lowest to highest for all items that are contained in this Intersection + */ + def leastToGreatest(implicit s: Successible[T]): Iterable[T] = { + val self = this + implicit val ord: Ordering[T] = s.ordering + // TODO https://github.com/twitter/algebird/issues/263 + new AbstractIterable[T] { + // We have to do this because the normal takeWhile causes OOM on big intervals + override def iterator: Iterator[T] = lower.toIterable.iterator.takeWhile(self.upper.contains(_)) + } + } + + def greatest(implicit p: Predecessible[T]): Option[T] = + upper.greatest.filter(lower.contains(_)(p.ordering)) + + /** + * Goes from highest to lowest for all items that are contained in this Intersection + */ + def greatestToLeast(implicit p: Predecessible[T]): Iterable[T] = { + val self = this + implicit val ord: Ordering[T] = p.ordering + // TODO https://github.com/twitter/algebird/issues/263 + new AbstractIterable[T] { + // We have to do this because the normal takeWhile causes OOM on big intervals + override def iterator: Iterator[T] = upper.toIterable.iterator.takeWhile(self.lower.contains(_)) + } + } + + /** + * Some intervals can actually be synonyms for empty: (0,0) for instance, contains nothing. This cannot be + * normalized to [a, b) form, thus we return an option Also, there are cases like [Int.MinValue, + * Int.MaxValue] that cannot are actually equivalent to Universe. The bottom line: if this returns None, it + * just means you can't express it this way, it does not mean it is empty or universe, etc... (there are + * other cases). + */ + def toLeftClosedRightOpen(implicit + s: Successible[T] + ): Option[Intersection[InclusiveLower, ExclusiveUpper, T]] = + for { + l <- lower.least + g <- upper.strictUpperBound if s.ordering.lt(l, g) + } yield Intersection(InclusiveLower(l), ExclusiveUpper(g)) +} diff --git a/algebird-core/src/main/scala-2.12+/InvariantAlgebrasCompat.scala b/algebird-core/src/main/scala-2.12+/InvariantAlgebrasCompat.scala new file mode 100644 index 000000000..464b1017f --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/InvariantAlgebrasCompat.scala @@ -0,0 +1,24 @@ +package com.twitter.algebird + +class InvariantSemigroup[T, U](val forward: T => U, val reverse: U => T)(implicit val semigroup: Semigroup[T]) + extends Semigroup[U] { + override def plus(l: U, r: U): U = + forward(semigroup.plus(reverse(l), reverse(r))) + override def sumOption(iter: TraversableOnce[U]): Option[U] = + semigroup.sumOption(iter.map(reverse)).map(forward) + + /* + * Note these work for the subclasses since in those cases semigroup + * will be the appropriate algebra. + */ + override val hashCode: Int = (forward, reverse, semigroup).hashCode + override def equals(that: Any): Boolean = + that match { + case r: InvariantSemigroup[?, ?] => + (hashCode == r.hashCode) && + (forward == r.forward) && + (reverse == r.reverse) && + (semigroup == r.semigroup) + case _ => false + } +} diff --git a/algebird-core/src/main/scala-2.12+/JMapMonoidsCompat.scala b/algebird-core/src/main/scala-2.12+/JMapMonoidsCompat.scala new file mode 100644 index 000000000..903f27ba5 --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/JMapMonoidsCompat.scala @@ -0,0 +1,52 @@ +package com.twitter.algebird + +import java.util.{HashMap => JHashMap, Map => JMap} +import scala.collection.JavaConverters._ + +/** + * Since maps are mutable, this always makes a full copy. Prefer scala immutable maps if you use scala + * immutable maps, this operation is much faster TODO extend this to Group, Ring + */ +class JMapMonoid[K, V: Semigroup] extends Monoid[JMap[K, V]] { + override lazy val zero: JHashMap[K, V] = new JHashMap[K, V](0) + + val nonZero: (V => Boolean) = implicitly[Semigroup[V]] match { + case mon: Monoid[?] => mon.isNonZero(_) + case _ => _ => true + } + + override def isNonZero(x: JMap[K, V]): Boolean = + !x.isEmpty && (implicitly[Semigroup[V]] match { + case mon: Monoid[?] => + x.values.asScala.exists(v => mon.isNonZero(v)) + case _ => true + }) + override def plus(x: JMap[K, V], y: JMap[K, V]): JHashMap[K, V] = { + val (big, small, bigOnLeft) = + if (x.size > y.size) { + (x, y, true) + } else { + (y, x, false) + } + val vsemi = implicitly[Semigroup[V]] + val result = new JHashMap[K, V](big.size + small.size) + result.putAll(big) + small.entrySet.asScala.foreach { kv => + val smallK = kv.getKey + val smallV = kv.getValue + if (big.containsKey(smallK)) { + val bigV = big.get(smallK) + val newV = + if (bigOnLeft) vsemi.plus(bigV, smallV) else vsemi.plus(smallV, bigV) + if (nonZero(newV)) + result.put(smallK, newV) + else + result.remove(smallK) + } else { + // No need to explicitly add with zero on V, just put in the small value + result.put(smallK, smallV) + } + } + result + } +} diff --git a/algebird-core/src/main/scala-2.12+/MapAlgebraCompat.scala b/algebird-core/src/main/scala-2.12+/MapAlgebraCompat.scala new file mode 100644 index 000000000..77511707e --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/MapAlgebraCompat.scala @@ -0,0 +1,75 @@ +package com.twitter.algebird + +import scala.collection.mutable.{Map => MMap} +import scala.collection.{Map => ScMap} +import scala.collection.compat._ + +abstract class GenericMapMonoid[K, V, M <: ScMap[K, V]](implicit val semigroup: Semigroup[V]) + extends Monoid[M] + with MapOperations[K, V, M] { + + val nonZero: (V => Boolean) = semigroup match { + case mon: Monoid[?] => mon.isNonZero(_) + case _ => _ => true + } + + override def isNonZero(x: M): Boolean = + !x.isEmpty && (semigroup match { + case mon: Monoid[?] => + x.valuesIterator.exists(v => mon.isNonZero(v)) + case _ => true + }) + + override def plus(x: M, y: M): M = { + // Scala maps can reuse internal structure, so don't copy just add into the bigger one: + // This really saves computation when adding lots of small maps into big ones (common) + val (big, small, bigOnLeft) = + if (x.size > y.size) { + (x, y, true) + } else { + (y, x, false) + } + small match { + // Mutable maps create new copies of the underlying data on add so don't use the + // handleImmutable method. + // Cannot have a None so 'get' is safe here. + case _: MMap[?, ?] => sumOption(Seq(big, small)).get + case _ => handleImmutable(big, small, bigOnLeft) + } + } + + private def handleImmutable(big: M, small: M, bigOnLeft: Boolean) = + small.foldLeft(big) { (oldMap, kv) => + val newV = big + .get(kv._1) + .map { bigV => + if (bigOnLeft) + semigroup.plus(bigV, kv._2) + else + semigroup.plus(kv._2, bigV) + } + .getOrElse(kv._2) + if (nonZero(newV)) + add(oldMap, kv._1 -> newV) + else + remove(oldMap, kv._1) + } + override def sumOption(items: TraversableOnce[M]): Option[M] = + if (items.iterator.isEmpty) None + else { + val mutable = MMap[K, V]() + items.iterator.foreach { m => + m.foreach { case (k, v) => + val oldVOpt = mutable.get(k) + // sorry for the micro optimization here: avoiding a closure + val newV = + if (oldVOpt.isEmpty) v else Semigroup.plus(oldVOpt.get, v) + if (nonZero(newV)) + mutable.update(k, newV) + else + mutable.remove(k) + } + } + Some(fromMutable(mutable)) + } +} diff --git a/algebird-core/src/main/scala-2.12+/ScanApplicativeCompat.scala b/algebird-core/src/main/scala-2.12+/ScanApplicativeCompat.scala new file mode 100644 index 000000000..dc49f701b --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/ScanApplicativeCompat.scala @@ -0,0 +1,16 @@ +package com.twitter.algebird + +class ScanApplicative[I] extends Applicative[Scan[I, _]] { + override def map[T, U](mt: Scan[I, T])(fn: T => U): Scan[I, U] = + mt.andThenPresent(fn) + + override def apply[T](v: T): Scan[I, T] = + Scan.const(v) + + override def join[T, U](mt: Scan[I, T], mu: Scan[I, U]): Scan[I, (T, U)] = + mt.join(mu) +} + +private[algebird] trait ScanApplicativeCompat { + implicit def applicative[I]: Applicative[Scan[I, _]] = new ScanApplicative[I] +} diff --git a/algebird-core/src/main/scala-2.12+/SpaceSaver.scala b/algebird-core/src/main/scala-2.12+/SpaceSaver.scala new file mode 100644 index 000000000..5f9eee7e6 --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/SpaceSaver.scala @@ -0,0 +1,296 @@ +package com.twitter.algebird + +import java.nio.ByteBuffer + +import scala.collection.immutable.SortedMap +import scala.util.{Failure, Success, Try} + +object SpaceSaver { + + /** + * Construct SpaceSaver with given capacity containing a single item. This is the public api to create a new + * SpaceSaver. + */ + def apply[T](capacity: Int, item: T): SpaceSaver[T] = SSOne(capacity, item) + + /** + * Construct SpaceSaver with given capacity containing a single item with provided exact count. This is the + * public api to create a new SpaceSaver. + */ + def apply[T](capacity: Int, item: T, count: Long): SpaceSaver[T] = + SSMany(capacity, Map(item -> ((count, 0L)))) + + private[algebird] val ordering = + Ordering.by[(?, (Long, Long)), (Long, Long)] { case (_, (count, err)) => + (-count, err) + } + + implicit def spaceSaverSemiGroup[T]: Semigroup[SpaceSaver[T]] = + new SpaceSaverSemigroup[T] + + /** + * Encodes the SpaceSaver as a sequence of bytes containing in order + * - 1 byte: 1/2 => 1 = SSOne, 2 = SSMany + * - 4 bytes: the capacity + * - N bytes: the item/counters (counters as length + N*(item size + item + 2 * counters) + */ + def toBytes[T](ss: SpaceSaver[T], tSerializer: T => Array[Byte]): Array[Byte] = + ss match { + case SSOne(capacity, item) => + val itemAsBytes = tSerializer(item) + val itemLength = itemAsBytes.length + // 1 for the type, 4 for capacity, 4 for itemAsBytes.length + val buffer = new Array[Byte](1 + 4 + 4 + itemLength) + ByteBuffer + .wrap(buffer) + .put(1: Byte) + .putInt(capacity) + .putInt(itemLength) + .put(itemAsBytes) + buffer + + case SSMany( + capacity, + counters, + _ + ) => // We do not care about the buckets are thery are created by SSMany.apply + val buffer = scala.collection.mutable.ArrayBuffer.newBuilder[Byte] + buffer += (2: Byte) + + var buff = ByteBuffer.allocate(4) + buff.putInt(capacity) + buffer ++= buff.array() + + buff = ByteBuffer.allocate(4) + buff.putInt(counters.size) + buffer ++= buff.array() + counters.foreach { case (item, (a, b)) => + val itemAsBytes = tSerializer(item) + + buff = ByteBuffer.allocate(4) + buff.putInt(itemAsBytes.length) + buffer ++= buff.array() + + buffer ++= itemAsBytes + + buff = ByteBuffer.allocate(8 * 2) + buff.putLong(a) + buff.putLong(b) + buffer ++= buff.array() + } + buffer.result().toArray + } + + // Make sure to be reversible so fromBytes(toBytes(x)) == x + def fromBytes[T](bytes: Array[Byte], tDeserializer: Array[Byte] => Try[T]): Try[SpaceSaver[T]] = + fromByteBuffer(ByteBuffer.wrap(bytes), buffer => tDeserializer(buffer.array())) + + def fromByteBuffer[T](bb: ByteBuffer, tDeserializer: ByteBuffer => Try[T]): Try[SpaceSaver[T]] = + Try { + bb.get.toInt match { + case 1 => + val capacity = bb.getInt + val itemLength = bb.getInt + val itemAsBytes = new Array[Byte](itemLength) + bb.get(itemAsBytes) + tDeserializer(ByteBuffer.wrap(itemAsBytes)).map(item => SSOne(capacity, item)) + case 2 => + val capacity = bb.getInt + + var countersToDeserialize = bb.getInt + val counters = scala.collection.mutable.Map.empty[T, (Long, Long)] + while (countersToDeserialize != 0) { + val itemLength = bb.getInt() + val itemAsBytes = new Array[Byte](itemLength) + bb.get(itemAsBytes) + val item = tDeserializer(ByteBuffer.wrap(itemAsBytes)) + + val a = bb.getLong + val b = bb.getLong + + item match { + case Failure(e) => return Failure(e) + case Success(i) => + counters += ((i, (a, b))) + } + + countersToDeserialize -= 1 + } + + Success(SSMany(capacity, counters.toMap)) + } + }.flatten +} + +/** + * Data structure used in the Space-Saving Algorithm to find the approximate most frequent and top-k elements. + * The algorithm is described in "Efficient Computation of Frequent and Top-k Elements in Data Streams". See + * here: www.cs.ucsb.edu/research/tech_reports/reports/2005-23.pdf In the paper the data structure is called + * StreamSummary but we chose to call it SpaceSaver instead. Note that the adaptation to hadoop and + * parallelization were not described in the article and have not been proven to be mathematically correct or + * preserve the guarantees or benefits of the algorithm. + */ +sealed abstract class SpaceSaver[T] { + import SpaceSaver.ordering + + /** + * Maximum number of counters to keep (parameter "m" in the research paper). + */ + def capacity: Int + + /** + * Current lowest value for count + */ + def min: Long + + /** + * Map of item to counter, where each counter consists of an observed count and possible over-estimation + * (error) + */ + def counters: Map[T, (Long, Long)] + + def ++(other: SpaceSaver[T]): SpaceSaver[T] + + /** + * returns the frequency estimate for the item + */ + def frequency(item: T): Approximate[Long] = { + val (count, err) = counters.getOrElse(item, (min, min)) + Approximate(count - err, count, count, 1.0) + } + + /** + * Get the elements that show up more than thres times. Returns sorted in descending order: (item, + * Approximate[Long], guaranteed) + */ + def mostFrequent(thres: Int): Seq[(T, Approximate[Long], Boolean)] = + counters.iterator + .filter { case (_, (count, _)) => count >= thres } + .toList + .sorted(ordering) + .map { case (item, (count, err)) => + (item, Approximate(count - err, count, count, 1.0), thres <= count - err) + } + + /** + * Get the top-k elements. Returns sorted in descending order: (item, Approximate[Long], guaranteed) + */ + def topK(k: Int): Seq[(T, Approximate[Long], Boolean)] = { + require(k < capacity) + val si = counters.toList + .sorted(ordering) + val siK = si.take(k) + val countKPlus1 = si.drop(k).headOption.map(_._2._1).getOrElse(0L) + siK.map { case (item, (count, err)) => + (item, Approximate(count - err, count, count, 1.0), countKPlus1 < count - err) + } + } + + /** + * Check consistency with other SpaceSaver, useful for testing. Returns boolean indicating if they are + * consistent + */ + def consistentWith(that: SpaceSaver[T]): Boolean = + (counters.keys ++ that.counters.keys).forall(item => (frequency(item) - that.frequency(item)) ~ 0) +} + +case class SSOne[T] private[algebird] (override val capacity: Int, item: T) extends SpaceSaver[T] { + require(capacity > 1) + + override def min: Long = 0L + + override def counters: Map[T, (Long, Long)] = Map(item -> ((1L, 1L))) + + override def ++(other: SpaceSaver[T]): SpaceSaver[T] = other match { + case other: SSOne[?] => SSMany(this).add(other) + case other: SSMany[?] => other.add(this) + } +} + +object SSMany { + private def bucketsFromCounters[T](counters: Map[T, (Long, Long)]): SortedMap[Long, Set[T]] = + SortedMap[Long, Set[T]]() ++ counters.groupBy(_._2._1).mapValues(_.keySet).toMap + + private[algebird] def apply[T](capacity: Int, counters: Map[T, (Long, Long)]): SSMany[T] = + SSMany(capacity, counters, bucketsFromCounters(counters)) + + private[algebird] def apply[T](one: SSOne[T]): SSMany[T] = + SSMany(one.capacity, Map(one.item -> ((1L, 0L))), SortedMap(1L -> Set(one.item))) +} + +case class SSMany[T] private ( + override val capacity: Int, + override val counters: Map[T, (Long, Long)], + buckets: SortedMap[Long, Set[T]] +) extends SpaceSaver[T] { + private val exact: Boolean = counters.size < capacity + + override val min: Long = if (counters.size < capacity) 0L else buckets.firstKey + + // item is already present and just needs to be bumped up one + private def bump(item: T) = { + val (count, err) = counters(item) + val counters1 = counters + (item -> ((count + 1L, err))) // increment by one + val currBucket = buckets(count) // current bucket + val buckets1 = { + if (currBucket.size == 1) // delete current bucket since it will be empty + buckets - count + else // remove item from current bucket + buckets + (count -> (currBucket - item)) + } + (count + 1L -> (buckets.getOrElse(count + 1L, Set()) + item)) + SSMany(capacity, counters1, buckets1) + } + + // lose one item to meet capacity constraint + private def loseOne = { + val firstBucket = buckets(buckets.firstKey) + val itemToLose = firstBucket.head + val counters1 = counters - itemToLose + val buckets1 = + if (firstBucket.size == 1) + buckets - min + else + buckets + (min -> (firstBucket - itemToLose)) + SSMany(capacity, counters1, buckets1) + } + + // introduce new item + private def introduce(item: T, count: Long, err: Long) = { + val counters1 = counters + (item -> ((count, err))) + val buckets1 = buckets + (count -> (buckets.getOrElse(count, Set()) + item)) + SSMany(capacity, counters1, buckets1) + } + + // add a single element + private[algebird] def add(x: SSOne[T]): SSMany[T] = { + require(x.capacity == capacity) + if (counters.contains(x.item)) + bump(x.item) + else + (if (exact) this else this.loseOne).introduce(x.item, min + 1L, min) + } + + // merge two stream summaries + private def merge(x: SSMany[T]): SSMany[T] = { + require(x.capacity == capacity) + val counters1 = Map() ++ + (counters.keySet ++ x.counters.keySet).toList + .map { key => + val (count1, err1) = counters.getOrElse(key, (min, min)) + val (count2, err2) = x.counters.getOrElse(key, (x.min, x.min)) + key -> ((count1 + count2, err1 + err2)) + } + .sorted(SpaceSaver.ordering) + .take(capacity) + SSMany(capacity, counters1) + } + + override def ++(other: SpaceSaver[T]): SpaceSaver[T] = other match { + case other: SSOne[?] => add(other) + case other: SSMany[?] => merge(other) + } +} + +class SpaceSaverSemigroup[T] extends Semigroup[SpaceSaver[T]] { + override def plus(x: SpaceSaver[T], y: SpaceSaver[T]): SpaceSaver[T] = x ++ y +} diff --git a/algebird-core/src/main/scala-2.12+/VectorSpace.scala b/algebird-core/src/main/scala-2.12+/VectorSpace.scala new file mode 100644 index 000000000..f8818600c --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/VectorSpace.scala @@ -0,0 +1,59 @@ +/* +Copyright 2012 Twitter, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.algebird + +import scala.annotation.implicitNotFound + +/** + * This class represents a vector space. For the required properties see: + * + * http://en.wikipedia.org/wiki/Vector_space#Definition + */ +object VectorSpace extends VectorSpaceOps with Implicits + +sealed trait VectorSpaceOps { + def scale[F, C[_]](v: F, c: C[F])(implicit vs: VectorSpace[F, C]): C[F] = + vs.scale(v, c) + def from[F, C[_]](scaleFn: (F, C[F]) => C[F])(implicit r: Ring[F], cGroup: Group[C[F]]): VectorSpace[F, C] = + new VectorSpace[F, C] { + override def ring: Ring[F] = r + override def group: Group[C[F]] = cGroup + override def scale(v: F, c: C[F]): C[F] = + if (r.isNonZero(v)) scaleFn(v, c) else cGroup.zero + } +} +private object VectorSpaceOps extends VectorSpaceOps + +sealed trait Implicits extends LowPrioImpicits { + implicit def indexedSeqSpace[T: Ring]: VectorSpace[T, IndexedSeq] = + VectorSpaceOps.from[T, IndexedSeq]((s, seq) => seq.map(Ring.times(s, _))) +} + +sealed trait LowPrioImpicits { + implicit def mapSpace[K, T: Ring]: VectorSpace[T, Map[K, _]] = + VectorSpaceOps.from[T, Map[K, _]] { (s, m) => + m.transform { case (_, v) => Ring.times(s, v) } + } +} + +@implicitNotFound(msg = "Cannot find VectorSpace type class for Container: ${C} and Ring: ${F}") +trait VectorSpace[F, C[_]] extends java.io.Serializable { + implicit def ring: Ring[F] + def field: Ring[F] = ring // this is for compatibility with older versions + implicit def group: Group[C[F]] + def scale(v: F, c: C[F]): C[F] +} diff --git a/algebird-core/src/main/scala-2.12+/monad/EitherMonad.scala b/algebird-core/src/main/scala-2.12+/monad/EitherMonad.scala new file mode 100644 index 000000000..00dce4c9b --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/monad/EitherMonad.scala @@ -0,0 +1,37 @@ +/* + Copyright 2013 Twitter, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +package com.twitter.algebird.monad + +import com.twitter.algebird.Monad + +// Monad for either, used for modeling Error where L is the type of the error +object EitherMonad { + class Error[L] extends Monad[Either[L, _]] { + override def apply[R](r: R): Right[L, R] = Right(r) + + override def flatMap[T, U](self: Either[L, T])(next: T => Either[L, U]): Either[L, U] = + self.right.flatMap(next) + + override def map[T, U](self: Either[L, T])(fn: T => U): Either[L, U] = + self.right.map(fn) + } + + implicit def monad[L]: Monad[Either[L, _]] = new Error[L] + + def assert[L](truth: Boolean, failure: => L): Either[L, Unit] = + if (truth) Right(()) else Left(failure) +} diff --git a/algebird-core/src/main/scala-2.12+/monad/Reader.scala b/algebird-core/src/main/scala-2.12+/monad/Reader.scala new file mode 100644 index 000000000..5eac6c4b8 --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/monad/Reader.scala @@ -0,0 +1,77 @@ +/* + Copyright 2013 Twitter, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +package com.twitter.algebird.monad + +import com.twitter.algebird.Monad + +// TODO this is general, move somewhere better + +// Reader Monad, represents a series of operations that mutate some environment +// type (the input to the function) + +sealed trait Reader[-Env, +T] { + def apply(env: Env): T + def flatMap[E1 <: Env, U](next: T => Reader[E1, U]): Reader[E1, U] = + FlatMappedReader[E1, T, U](this, next) + def map[U](thatFn: T => U): Reader[Env, U] = + FlatMappedReader(this, (t: T) => ConstantReader(thatFn(t))) +} + +final case class ConstantReader[+T](get: T) extends Reader[Any, T] { + override def apply(env: Any): T = get + override def map[U](fn: T => U): ConstantReader[U] = ConstantReader(fn(get)) + override def flatMap[E1 <: Any, U](next: T => Reader[E1, U]): Reader[E1, U] = + next(get) +} +final case class ReaderFn[E, +T](fn: E => T) extends Reader[E, T] { + override def apply(env: E): T = fn(env) +} +final case class FlatMappedReader[E, U, +T](first: Reader[E, U], fn: U => Reader[E, T]) extends Reader[E, T] { + override def apply(env: E): T = { + @annotation.tailrec + def loop(r: Reader[E, Any], stack: List[(Any) => Reader[E, Any]]): Any = + r match { + case ConstantReader(get) => + stack match { + case head :: tail => loop(head(get), tail) + case Nil => get + } + case ReaderFn(fn) => + stack match { + case head :: tail => loop(head(fn(env)), tail) + case Nil => fn(env) + } + case FlatMappedReader(first, nextFn) => + loop(first, nextFn.asInstanceOf[Any => Reader[E, Any]] :: stack) + } + loop(first, List(fn.asInstanceOf[(Any) => Reader[E, Any]])).asInstanceOf[T] + } +} + +object Reader { + def const[T](t: T): Reader[Any, T] = ConstantReader(t) + implicit def apply[E, T](fn: (E) => T): Reader[E, T] = ReaderFn(fn) + + class ReaderM[Env] extends Monad[Reader[Env, _]] { + override def apply[T](t: T): ConstantReader[T] = ConstantReader(t) + override def flatMap[T, U](self: Reader[Env, T])(next: T => Reader[Env, U]): Reader[Env, U] = + self.flatMap(next) + override def map[T, U](self: Reader[Env, T])(fn: T => U): Reader[Env, U] = self.map(fn) + } + + implicit def monad[Env]: Monad[Reader[Env, _]] = new ReaderM[Env] +} diff --git a/algebird-core/src/main/scala-2.12+/monad/StateWithError.scala b/algebird-core/src/main/scala-2.12+/monad/StateWithError.scala new file mode 100644 index 000000000..e47503eb8 --- /dev/null +++ b/algebird-core/src/main/scala-2.12+/monad/StateWithError.scala @@ -0,0 +1,131 @@ +/* + Copyright 2013 Twitter, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +package com.twitter.algebird.monad + +import com.twitter.algebird.{Monad, Semigroup} + +/** + * Monad to handle mutating input state and possible failures. This is used to interact in the planning phase + * with existing mutable APIs (like storm or cascading), but retain the ability to compose carefully. + */ +sealed trait StateWithError[S, +F, +T] { + def join[F1 >: F, U]( + that: StateWithError[S, F1, U], + mergeErr: (F1, F1) => F1, + mergeState: (S, S) => S + ): StateWithError[S, F1, (T, U)] = + join(that)(Semigroup.from(mergeErr), Semigroup.from(mergeState)) + + def join[F1 >: F, U](that: StateWithError[S, F1, U])(implicit + sgf: Semigroup[F1], + sgs: Semigroup[S] + ): // TODO: deep joins could blow the stack, not yet using trampoline here + StateWithError[S, F1, (T, U)] = + StateFn { (requested: S) => + (run(requested), that.run(requested)) match { + case (Right((s1, r1)), Right((s2, r2))) => + Right((sgs.plus(s1, s2), (r1, r2))) + case (Left(err1), Left(err2)) => + Left(sgf.plus(err1, err2)) // Our earlier is not ready + case (Left(err), _) => Left(err) + case (_, Left(err)) => Left(err) + } + } + + def apply(state: S): Either[F, (S, T)] = run(state) + + def run(state: S): Either[F, (S, T)] + + def flatMap[F1 >: F, U](next: T => StateWithError[S, F1, U]): StateWithError[S, F1, U] = + FlatMappedState(this, next) + + def map[U](fn: (T) => U): StateWithError[S, F, U] = + FlatMappedState(this, (t: T) => StateWithError.const(fn(t))) +} + +/** Simple wrapper of a function in the Monad */ +final case class StateFn[S, F, T](fn: S => Either[F, (S, T)]) extends StateWithError[S, F, T] { + override def run(state: S): Either[F, (S, T)] = fn(state) +} + +/** + * A Trampolining instance that should prevent stack overflow at the expense of performance + */ +final case class FlatMappedState[S, F, T, U](start: StateWithError[S, F, T], fn: T => StateWithError[S, F, U]) + extends StateWithError[S, F, U] { + override def run(state: S): Either[F, (S, U)] = { + @annotation.tailrec + def loop(inState: S, st: StateWithError[S, F, Any], stack: List[Any => StateWithError[S, F, Any]]): Any = + st match { + case StateFn(fn) => + fn(inState) match { + case err @ Left(_) => err // bail at first error + case noError @ Right((newState, out)) => + stack match { + case head :: tailStack => loop(newState, head(out), tailStack) + case Nil => noError // recursion ends + } + } + case FlatMappedState(st, next) => + loop(inState, st, next.asInstanceOf[Any => StateWithError[S, F, Any]] :: stack) + } + loop(state, this, Nil).asInstanceOf[Either[F, (S, U)]] + } +} + +object StateWithError { + def getState[S]: StateWithError[S, Nothing, S] = + StateFn((state: S) => Right((state, state))) + def putState[S](newState: S): StateWithError[S, Nothing, Unit] = + StateFn((_: S) => Right((newState, ()))) + def swapState[S](newState: S): StateWithError[S, Nothing, S] = + StateFn((old: S) => Right((newState, old))) + + def const[S, T](t: T): StateWithError[S, Nothing, T] = + StateFn((state: S) => Right((state, t))) + def lazyVal[S, T](t: => T): StateWithError[S, Nothing, T] = + StateFn((state: S) => Right((state, t))) + def failure[S, F](f: F): StateWithError[S, F, Nothing] = + StateFn(_ => Left(f)) + + /** + * Use like fromEither[Int](Right("good")) to get a constant Either in the monad + */ + def fromEither[S]: ConstantStateMaker[S] = new ConstantStateMaker[S] + class ConstantStateMaker[S] { + def apply[F, T](either: Either[F, T]): StateWithError[S, F, T] = { (s: S) => either.right.map((s, _)) } + } + + class FunctionLifter[S] { + def apply[I, F, T](fn: I => Either[F, T]): (I => StateWithError[S, F, T]) = { (i: I) => + StateFn((s: S) => fn(i).right.map((s, _))) + } + } + // TODO this should move to Monad and work for any Monad + def toKleisli[S]: FunctionLifter[S] = new FunctionLifter[S] + + implicit def apply[S, F, T](fn: S => Either[F, (S, T)]): StateWithError[S, F, T] = StateFn(fn) + implicit def monad[S, F]: Monad[StateWithError[S, F, _]] = new StateFMonad[F, S] + + class StateFMonad[F, S] extends Monad[StateWithError[S, F, _]] { + override def apply[T](const: T): StateWithError[S, Nothing, T] = { (s: S) => Right((s, const)) } + override def flatMap[T, U]( + earlier: StateWithError[S, F, T] + )(next: T => StateWithError[S, F, U]): StateWithError[S, F, U] = + earlier.flatMap(next) + } +} diff --git a/algebird-core/src/main/scala-2.13+/com/twitter/algebird/macros/MacroCompat.scala b/algebird-core/src/main/scala-2.13+/com/twitter/algebird/macros/MacroCompat.scala deleted file mode 100644 index a478977dc..000000000 --- a/algebird-core/src/main/scala-2.13+/com/twitter/algebird/macros/MacroCompat.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.algebird.macros - -import scala.reflect.macros.whitebox - -private[algebird] object MacroCompat { - - type Context = whitebox.Context - - def normalize(c: Context)(tpe: c.universe.Type) = tpe.etaExpand - - def declarations(c: Context)(tpe: c.universe.Type) = tpe.decls - - def companionSymbol[T](c: Context)(typeSymbol: c.universe.Symbol) = typeSymbol.companion - - def typeName(c: Context)(s: String): c.universe.TypeName = c.universe.TypeName(s) - - def termName(c: Context)(s: String): c.universe.TermName = c.universe.TermName(s) - -} diff --git a/algebird-core/src/main/scala-2.13+/com/twitter/algebird/compat.scala b/algebird-core/src/main/scala-2.13/com/twitter/algebird/compat.scala similarity index 100% rename from algebird-core/src/main/scala-2.13+/com/twitter/algebird/compat.scala rename to algebird-core/src/main/scala-2.13/com/twitter/algebird/compat.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/AdaptiveVector.scala b/algebird-core/src/main/scala-2/com/twitter/algebird/AdaptiveVector.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/AdaptiveVector.scala rename to algebird-core/src/main/scala-2/com/twitter/algebird/AdaptiveVector.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/macros/Cuber.scala b/algebird-core/src/main/scala-2/com/twitter/algebird/macros/Cuber.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/macros/Cuber.scala rename to algebird-core/src/main/scala-2/com/twitter/algebird/macros/Cuber.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/macros/GroupMacro.scala b/algebird-core/src/main/scala-2/com/twitter/algebird/macros/GroupMacro.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/macros/GroupMacro.scala rename to algebird-core/src/main/scala-2/com/twitter/algebird/macros/GroupMacro.scala diff --git a/algebird-core/src/main/scala-2.12-/com/twitter/algebird/macros/MacroCompat.scala b/algebird-core/src/main/scala-2/com/twitter/algebird/macros/MacroCompat.scala similarity index 100% rename from algebird-core/src/main/scala-2.12-/com/twitter/algebird/macros/MacroCompat.scala rename to algebird-core/src/main/scala-2/com/twitter/algebird/macros/MacroCompat.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/macros/MonoidMacro.scala b/algebird-core/src/main/scala-2/com/twitter/algebird/macros/MonoidMacro.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/macros/MonoidMacro.scala rename to algebird-core/src/main/scala-2/com/twitter/algebird/macros/MonoidMacro.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/macros/RingMacro.scala b/algebird-core/src/main/scala-2/com/twitter/algebird/macros/RingMacro.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/macros/RingMacro.scala rename to algebird-core/src/main/scala-2/com/twitter/algebird/macros/RingMacro.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/macros/Roller.scala b/algebird-core/src/main/scala-2/com/twitter/algebird/macros/Roller.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/macros/Roller.scala rename to algebird-core/src/main/scala-2/com/twitter/algebird/macros/Roller.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/macros/SemigroupMacro.scala b/algebird-core/src/main/scala-2/com/twitter/algebird/macros/SemigroupMacro.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/macros/SemigroupMacro.scala rename to algebird-core/src/main/scala-2/com/twitter/algebird/macros/SemigroupMacro.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/macros/caseclass.scala b/algebird-core/src/main/scala-2/com/twitter/algebird/macros/caseclass.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/macros/caseclass.scala rename to algebird-core/src/main/scala-2/com/twitter/algebird/macros/caseclass.scala diff --git a/algebird-core/src/main/scala/com/twitter/algebird/macros/package.scala b/algebird-core/src/main/scala-2/com/twitter/algebird/macros/package.scala similarity index 100% rename from algebird-core/src/main/scala/com/twitter/algebird/macros/package.scala rename to algebird-core/src/main/scala-2/com/twitter/algebird/macros/package.scala diff --git a/algebird-core/src/main/scala-3/com/twitter/algebird/AdaptiveVector.scala b/algebird-core/src/main/scala-3/com/twitter/algebird/AdaptiveVector.scala new file mode 100644 index 000000000..e066849c2 --- /dev/null +++ b/algebird-core/src/main/scala-3/com/twitter/algebird/AdaptiveVector.scala @@ -0,0 +1,252 @@ +/* +Copyright 2012 Twitter, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.algebird + +import com.twitter.algebird.collections.compat._ +import scala.annotation.tailrec + +/** + * Some functions to create or convert AdaptiveVectors + */ +object AdaptiveVector { + + /** + * When density >= this value * size, we switch to dense vectors + */ + val THRESHOLD: Double = 0.25 + def fill[V](size: Int)(sparse: V): AdaptiveVector[V] = + SparseVector(Map.empty[Int, V], sparse, size) + + def fromVector[V](v: Vector[V], sparseVal: V): AdaptiveVector[V] = + if (v.size == 0) { + fill[V](0)(sparseVal) + } else { + val denseCount = v.count(_ != sparseVal) + val sz = v.size + if (denseCount < sz * THRESHOLD) + SparseVector(toMap(v, sparseVal), sparseVal, sz) + else + DenseVector(v, sparseVal, denseCount) + } + def fromMap[V](m: Map[Int, V], sparseVal: V, sizeOfDense: Int): AdaptiveVector[V] = + if (m.size == 0) { + fill[V](sizeOfDense)(sparseVal) + } else { + val maxIdx = m.keys.max + require(maxIdx < sizeOfDense, "Max key (" + maxIdx + ") exceeds valid for size (" + sizeOfDense + ")") + val denseCount = m.count(_._2 != sparseVal) + if (denseCount < sizeOfDense * THRESHOLD) + SparseVector(m, sparseVal, sizeOfDense) + else + DenseVector(toVector(m, sparseVal, sizeOfDense), sparseVal, denseCount) + } + + def toMap[V](v: AdaptiveVector[V]): Map[Int, V] = + v match { + case DenseVector(is, sv, _) => toMap(is, sv) + case SparseVector(m, _, _) => m + } + + def toMap[V](iseq: IndexedSeq[V], sparse: V): Map[Int, V] = + iseq.view.zipWithIndex.filter(_._1 != sparse).map(_.swap).toMap + + def toVector[V](m: Map[Int, V], sparse: V, size: Int): Vector[V] = { + // Mutable local variable to optimize performance + import scala.collection.mutable.Buffer + val buf = Buffer.fill[V](size)(sparse) + m.foreach { case (idx, v) => buf(idx) = v } + Vector.from(buf) + } + + def toVector[V](v: AdaptiveVector[V]): Vector[V] = + v match { + case DenseVector(is, _, _) => is + case SparseVector(m, v, sz) => toVector(m, v, sz) + } + + private def withSparse[V](v: AdaptiveVector[V], sv: V): AdaptiveVector[V] = + if (v.sparseValue == sv) v + else fromVector(toVector(v), sv) + + private class AVSemigroup[V: Semigroup] extends Semigroup[AdaptiveVector[V]] { + private def valueIsNonZero(v: V): Boolean = implicitly[Semigroup[V]] match { + case m: Monoid[?] => m.isNonZero(v) + case _ => true + } + + override def plus(left: AdaptiveVector[V], right: AdaptiveVector[V]): AdaptiveVector[V] = + if (left.sparseValue != right.sparseValue) { + if (left.denseCount > right.denseCount) + plus(withSparse(left, right.sparseValue), right) + else plus(left, withSparse(right, left.sparseValue)) + } else { + // they have the same sparse value + val maxSize = Ordering[Int].max(left.size, right.size) + (left, right) match { + case (DenseVector(lv, ls, _), DenseVector(rv, _, _)) => + val vec = Semigroup.plus[IndexedSeq[V]](lv, rv) match { + case v: Vector[?] => v.asInstanceOf[Vector[V]] + case notV => Vector.from(notV) + } + fromVector(vec, ls) + + case _ if valueIsNonZero(left.sparseValue) => + fromVector( + Vector.from(Semigroup.plus(toVector(left): IndexedSeq[V], toVector(right): IndexedSeq[V])), + left.sparseValue + ) + case _ => // sparse is zero: + fromMap(Semigroup.plus(toMap(left), toMap(right)), left.sparseValue, maxSize) + } + } + } + private class AVMonoid[V: Monoid] extends AVSemigroup[V] with Monoid[AdaptiveVector[V]] { + override val zero: AdaptiveVector[V] = AdaptiveVector.fill[V](0)(Monoid.zero[V]) + override def isNonZero(v: AdaptiveVector[V]): Boolean = !isZero(v) + + def isZero(v: AdaptiveVector[V]): Boolean = (v.size == 0) || { + val sparseAreZero = + if (Monoid.isNonZero(v.sparseValue)) v.denseCount == v.size else true + sparseAreZero && + v.denseIterator.forall(idxv => !Monoid.isNonZero(idxv._2)) + } + } + private class AVGroup[V: Group] extends AVMonoid[V] with Group[AdaptiveVector[V]] { + override def negate(v: AdaptiveVector[V]): AdaptiveVector[V] = + fromVector(toVector(v).map(Group.negate(_)), Group.negate(v.sparseValue)) + } + + implicit def semigroup[V: Semigroup]: Semigroup[AdaptiveVector[V]] = + new AVSemigroup[V] + implicit def monoid[V: Monoid]: Monoid[AdaptiveVector[V]] = new AVMonoid[V] + implicit def group[V: Group]: Group[AdaptiveVector[V]] = new AVGroup[V] + + /* + * Equality when considering only the dense values (so size doesn't matter) + */ + def denseEquiv[V: Equiv]: Equiv[AdaptiveVector[V]] = + Equiv.fromFunction[AdaptiveVector[V]] { (l, r) => + val (lit, rit) = (l.denseIterator, r.denseIterator) + @tailrec + def iteq: Boolean = + (lit.hasNext, rit.hasNext) match { + case (true, true) => + val (lnext, rnext) = (lit.next(), rit.next()) + if (lnext._1 == rnext._1 && Equiv[V].equiv(lnext._2, rnext._2)) + iteq + else + false + case (false, false) => true + case _ => false + } + Equiv[V].equiv(l.sparseValue, r.sparseValue) && iteq + } + + implicit def equiv[V: Equiv]: Equiv[AdaptiveVector[V]] = + Equiv.fromFunction[AdaptiveVector[V]] { (l, r) => + (l.size == r.size) && (denseEquiv[V].equiv(l, r) || + toVector(l).view.zip(toVector(r)).forall { case (lv, rv) => + Equiv[V].equiv(lv, rv) + }) + } +} + +/** + * An IndexedSeq that automatically switches representation between dense and sparse depending on sparsity + * Should be an efficient representation for all sizes, and it should not be necessary to special case + * immutable algebras based on the sparsity of the vectors. + */ +sealed trait AdaptiveVector[V] extends IndexedSeq[V] { + override def length: Int = size + def sparseValue: V + + /** How many items are not sparse */ + def denseCount: Int + // def size: Int + override def apply(idx: Int): V + def updated(idx: Int, v: V): AdaptiveVector[V] + + /** Grow by adding count sparse values to the end */ + def extend(count: Int): AdaptiveVector[V] + + /** Iterator of indices and values of all non-sparse values */ + def denseIterator: Iterator[(Int, V)] + /* + * Note that IndexedSeq provides hashCode and equals that + * work correctly based on length and apply. + */ +} + +case class DenseVector[V](iseq: Vector[V], override val sparseValue: V, override val denseCount: Int) + extends AdaptiveVector[V] { + + override def length: Int = iseq.size + override def apply(idx: Int): V = iseq(idx) + override def updated(idx: Int, v: V): AdaptiveVector[V] = { + val oldIsSparse = if (iseq(idx) == sparseValue) 1 else 0 + val newIsSparse = if (v == sparseValue) 1 else 0 + val newCount = denseCount - newIsSparse + oldIsSparse + if (denseCount < size * AdaptiveVector.THRESHOLD) { + // Go sparse + SparseVector(AdaptiveVector.toMap(iseq, sparseValue), sparseValue, size) + } else { + DenseVector(iseq.updated(idx, v), sparseValue, newCount) + } + } + override def extend(cnt: Int): AdaptiveVector[V] = { + val newSize = size + cnt + if (denseCount < newSize * AdaptiveVector.THRESHOLD) { + // Go sparse + SparseVector(AdaptiveVector.toMap(iseq, sparseValue), sparseValue, newSize) + } else { + // Stay dense + DenseVector(iseq ++ Vector.fill(cnt)(sparseValue), sparseValue, denseCount) + } + } + + override def denseIterator: Iterator[(Int, V)] = + iseq.view.zipWithIndex + .filter(_._1 != sparseValue) + .map(_.swap) + .iterator +} + +case class SparseVector[V](map: Map[Int, V], override val sparseValue: V, override val length: Int) + extends AdaptiveVector[V] { + + override def denseCount: Int = map.size + override def apply(idx: Int): V = { + require(idx >= 0 && idx < size, "Index out of range") + map.getOrElse(idx, sparseValue) + } + override def updated(idx: Int, v: V): AdaptiveVector[V] = + if (v == sparseValue) { + SparseVector(map - idx, sparseValue, size) + } else { + val newM = map + (idx -> v) + if (newM.size >= size * AdaptiveVector.THRESHOLD) { + // Go dense: + DenseVector(AdaptiveVector.toVector(newM, sparseValue, size), sparseValue, newM.size) + } else { + SparseVector(newM, sparseValue, size) + } + } + override def extend(cnt: Int): SparseVector[V] = SparseVector(map, sparseValue, size + cnt) + + private lazy val sortedList = map.toList.sortBy(_._1) + override def denseIterator: Iterator[(Int, V)] = sortedList.iterator +} diff --git a/algebird-core/src/main/scala-3/com/twitter/algebird/compat.scala b/algebird-core/src/main/scala-3/com/twitter/algebird/compat.scala new file mode 100644 index 000000000..43e1b18fc --- /dev/null +++ b/algebird-core/src/main/scala-3/com/twitter/algebird/compat.scala @@ -0,0 +1,36 @@ +package com.twitter.algebird + +import scala.collection.Factory +import scala.collection.mutable.{Builder, Map => MMap} + +private[algebird] class MutableBackedMap[K, V](val backingMap: MMap[K, V]) + extends Map[K, V] + with java.io.Serializable { + override def get(key: K): Option[V] = backingMap.get(key) + + override def iterator: Iterator[(K, V)] = backingMap.iterator + + override def updated[B1 >: V](k: K, v: B1): Map[K, B1] = backingMap.toMap + (k -> v) + + override def removed(key: K): Map[K, V] = backingMap.toMap - key +} + +private[algebird] trait CompatFold { + + /** + * Simple Fold that collects elements into a container. + */ + def container[I, C[_]](implicit cbf: Factory[I, C[I]]): Fold[I, C[I]] = + Fold.foldMutable[Builder[I, C[I]], I, C[I]]( + { case (b, i) => b += i }, + _ => cbf.newBuilder, + _.result + ) +} + +private[algebird] trait CompatDecayedVector { + // This is the default monoid that never thresholds. + // If you want to set a specific accuracy you need to implicitly override this +// implicit def monoid[F, C[_]](implicit vs: VectorSpace[F, C], metric: Metric[C[F]]):Monoid[DecayedVector[C]] = +// DecayedVector.monoidWithEpsilon(-1.0) +} diff --git a/algebird-core/src/main/scala-3/com/twitter/algebird/macros/Cuber.scala b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/Cuber.scala new file mode 100644 index 000000000..e7a3b69a5 --- /dev/null +++ b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/Cuber.scala @@ -0,0 +1,84 @@ +package com.twitter.algebird.macros +import scala.compiletime.summonInline +import scala.deriving.Mirror +import scala.quoted.Expr +import scala.quoted.Quotes +import scala.quoted.Type +import scala.runtime.Tuples + +/** + * "Cubes" a case class or tuple, i.e. for a tuple of type (T1, T2, ... , TN) generates all 2^N possible + * combinations of type (Option[T1], Option[T2], ... , Option[TN]). + * + * This is useful for comparing some metric across all possible subsets. For example, suppose we have a set of + * people represented as case class Person(gender: String, age: Int, height: Double) and we want to know the + * average height of + * - people, grouped by gender and age + * - people, grouped by only gender + * - people, grouped by only age + * - all people + * + * Then we could do > import com.twitter.algebird.macros.Cuber.cuber > val people: List[People] > val + * averageHeights: Map[(Option[String], Option[Int]), Double] = > people.flatMap { p => cuber((p.gender, + * p.age)).map((_,p)) } > .groupBy(_._1) > .mapValues { xs => val heights = xs.map(_.height); heights.sum / + * heights.length } + */ +trait Cuber[I]: + type K + def apply(in: I): TraversableOnce[K] + +object Cuber extends MacroHelper: + implicit inline def cuber[T]: Cuber[T] = ${ deriveCuberImpl[T] } + inline def derived[T]: Cuber[T] = ${ deriveCuberImpl[T] } + def deriveCuberImpl[T: Type](using q: Quotes): Expr[Cuber[T]] = + import q.reflect.* + val tname = TypeRepr.of[T].typeSymbol.name + val ev: Expr[Mirror.Of[T]] = Expr.summon[Mirror.Of[T]] match + case None => + report.errorAndAbort(s"unable to derive Cuber instance for ${tname}") + case Some(expr) => expr + + ev match + case '{ + $m: Mirror.ProductOf[T] { type MirroredElemTypes = elementTypes } + } => + val caseclassFields = TypeRepr.of[T].typeSymbol.caseFields + if caseclassFields.length > 22 then + report.errorAndAbort( + s"Cannot create Cuber for $tname because it has more than 22 parameters." + ) + val tupleNTyped = genTupleNTyped(caseclassFields) + val somes: List[Symbol] = genSomes(caseclassFields) + val idents = Expr.ofList(somes.map(sym => Ident(sym.termRef).asExpr)) + + val arity = Expr(caseclassFields.length) + + def options(i: Expr[Int]): Expr[Seq[Option[Any]]] = + '{ + (1 to ${ arity }).map(index => + if ((1 << { index - 1 }) & ${ i }) == 0 then None + else ${ idents }(index - 1).asInstanceOf[Option[Any]] + ) + } + + def blc(e: Expr[T]): Expr[Any] = Block( + somesStmt[T](somes, caseclassFields, e), + '{ + (0 until (1 << ${ arity })).map { i => + Tuple.fromArray(${ options('{ i }) }.toArray) + } + }.asTerm + ).asExpr + + tupleNTyped.tpe.widen.asType match + case '[t] => + '{ + + new Cuber[T]: + type K = t + override def apply(in: T): TraversableOnce[K] = + ${ blc('{ in }) }.asInstanceOf[TraversableOnce[K]] + } + + case '{ $m: Mirror.SumOf[T] { type MirroredElemTypes = elementTypes } } => + report.errorAndAbort(s"unable to derive Cuber for ${tname}") diff --git a/algebird-core/src/main/scala-3/com/twitter/algebird/macros/GroupMacro.scala b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/GroupMacro.scala new file mode 100644 index 000000000..9ab64cec3 --- /dev/null +++ b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/GroupMacro.scala @@ -0,0 +1,76 @@ +package com.twitter.algebird.macros + +import com.twitter.algebird.* + +import scala.compiletime.summonInline +import scala.deriving.Mirror +import scala.quoted.Expr +import scala.quoted.Quotes +import scala.quoted.Type +import scala.runtime.Tuples +object GroupMacro: + + def summonAll[T: Type](using q: Quotes): List[Expr[Group[?]]] = + Type.of[T] match + case '[tpe *: tpes] => + Expr.summon[Group[tpe]] match + case None => derivedGroup[tpe] :: summonAll[tpes] + case Some(inst) => inst :: summonAll[tpes] + case '[EmptyTuple] => Nil + + def derivedGroup[T: Type](using q: Quotes): Expr[Group[T]] = + import q.reflect.* + val ev: Expr[Mirror.Of[T]] = Expr.summon[Mirror.Of[T]] match + case None => + val tname = TypeRepr.of[T].typeSymbol.name + report.errorAndAbort(s"unable to find or derive Group instance for ${tname}") + case Some(expr) => expr + + ev match + case '{ $m: Mirror.ProductOf[T] { type MirroredElemTypes = elementTypes } } => + val insts = summonAll[elementTypes] + val instsAsExpr = Expr.ofTupleFromSeq(insts) + + '{ + new Group[T]: + override def zero: T = + val t = MonoidMacro.applyZeros(${ instsAsExpr }).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + override def plus(x: T, y: T): T = + val args = Tuple + .fromProduct(x.asInstanceOf[Product]) + .zip(Tuple.fromProduct(y.asInstanceOf[Product])) + .zip(${ instsAsExpr }) + .asInstanceOf[Tuple] + val t = SemigroupMacro.applyPlus(args).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + override def minus(x: T, y: T): T = + val args = Tuple + .fromProduct(x.asInstanceOf[Product]) + .zip(Tuple.fromProduct(y.asInstanceOf[Product])) + .zip(${ instsAsExpr }) + .asInstanceOf[Tuple] + val t = applyMinus(args).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + override def negate(x: T): T = + val args = Tuple.fromProduct(x.asInstanceOf[Product]).zip(${ instsAsExpr }).asInstanceOf[Tuple] + val t = applyNegates(args).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + + } + + case '{ $m: Mirror.SumOf[T] { type MirroredElemTypes = elementTypes } } => + val tname = TypeRepr.of[T].typeSymbol.name + report.errorAndAbort(s"unable to derive Group for ${tname}") + private[macros] def applyNegates(a: Tuple): Tuple = + a match + case EmptyTuple => EmptyTuple + case ((x, inst): ((?, Group[?]))) *: ts => + inst.asInstanceOf[Group[Any]].negate(x) *: applyNegates(ts) + case _ => ??? + private[macros] def applyMinus(a: Tuple): Tuple = + a match + case EmptyTuple => EmptyTuple + case (((x, y), inst): ((?, ?), Group[?])) *: ts => + inst.asInstanceOf[Group[Any]].minus(x, y) *: applyMinus(ts) + case _ => ??? diff --git a/algebird-core/src/main/scala-3/com/twitter/algebird/macros/MacroHelper.scala b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/MacroHelper.scala new file mode 100644 index 000000000..10bef6532 --- /dev/null +++ b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/MacroHelper.scala @@ -0,0 +1,58 @@ +package com.twitter.algebird.macros +import scala.quoted.Quotes +import scala.quoted.Expr + +trait MacroHelper: + inline def genTupleNTyped(using q: Quotes)( + caseclassFields: List[q.reflect.Symbol] + ): q.reflect.Applied = + import q.reflect.* + val optTpes = caseclassFields + .map(sym => + Applied( + TypeIdent(TypeRepr.of[Option].typeSymbol), + List(sym.termRef.typeSymbol.tree) + ) + ) + .toList + val tupleClazz = + Symbol.classSymbol(s"scala.Tuple${caseclassFields.length}") + val tupleNIdent = Ident(tupleClazz.termRef) + val tupleN = TypeIdent(tupleNIdent.symbol) + Applied(tupleN, optTpes) + def genSomes(using q: Quotes)( + caseclassFields: List[q.reflect.Symbol] + ): List[q.reflect.Symbol] = + import q.reflect.* + caseclassFields.map(sym => + sym.termRef.widen.asType match + case '[t] => + Symbol.newVal( + Symbol.spliceOwner, + sym.name, + TypeRepr.of[Option[t]], + Flags.EmptyFlags, + Symbol.noSymbol + ) + ) + def somesStmt[T](using q: Quotes)( + somes: List[q.reflect.Symbol], + caseclassFields: List[q.reflect.Symbol], + caseclass: Expr[T] + ): List[q.reflect.ValDef] = + import q.reflect.* + somes + .zip(caseclassFields) + .map((someSym, field) => + ValDef( + someSym, + Some { + field.termRef.widen.asType match + case '[t] => + val value = Select(caseclass.asTerm, field).asExprOf[t] + '{ Some(${ value }) }.asTerm + } + ) + ) + +private[algebird] object MacroCompat diff --git a/algebird-core/src/main/scala-3/com/twitter/algebird/macros/MonoidMacro.scala b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/MonoidMacro.scala new file mode 100644 index 000000000..9cc3c2e73 --- /dev/null +++ b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/MonoidMacro.scala @@ -0,0 +1,58 @@ +package com.twitter.algebird.macros + +import com.twitter.algebird.* + +import scala.compiletime.summonInline +import scala.deriving.Mirror +import scala.quoted.Expr +import scala.quoted.Quotes +import scala.quoted.Type +import scala.runtime.Tuples +object MonoidMacro: + + def summonAll[T: Type](using q: Quotes): List[Expr[Monoid[?]]] = + Type.of[T] match + case '[tpe *: tpes] => + Expr.summon[Monoid[tpe]] match + case None => derivedMonoid[tpe] :: summonAll[tpes] + case Some(inst) => inst :: summonAll[tpes] + case '[EmptyTuple] => Nil + + def derivedMonoid[T: Type](using q: Quotes): Expr[Monoid[T]] = + import q.reflect.* + val ev: Expr[Mirror.Of[T]] = Expr.summon[Mirror.Of[T]] match + case None => + val tname = TypeRepr.of[T].typeSymbol.name + report.errorAndAbort(s"unable to find or derive Monoid instance for ${tname}") + case Some(expr) => expr + + ev match + case '{ $m: Mirror.ProductOf[T] { type MirroredElemTypes = elementTypes } } => + val insts = summonAll[elementTypes] + val instsAsExpr = Expr.ofTupleFromSeq(insts) + + '{ + new Monoid[T]: + override def zero: T = + val t = applyZeros(${ instsAsExpr }).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + override def plus(x: T, y: T): T = + val args = Tuple + .fromProduct(x.asInstanceOf[Product]) + .zip(Tuple.fromProduct(y.asInstanceOf[Product])) + .zip(${ instsAsExpr }) + .asInstanceOf[Tuple] + val t = SemigroupMacro.applyPlus(args).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + + } + + case '{ $m: Mirror.SumOf[T] { type MirroredElemTypes = elementTypes } } => + val tname = TypeRepr.of[T].typeSymbol.name + report.errorAndAbort(s"unable to derive Monoid for ${tname}") + + private[macros] def applyZeros(a: Tuple): Tuple = + a match + case EmptyTuple => EmptyTuple + case inst *: ts => + inst.asInstanceOf[Monoid[Any]].zero *: applyZeros(ts) diff --git a/algebird-core/src/main/scala-3/com/twitter/algebird/macros/RingMacro.scala b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/RingMacro.scala new file mode 100644 index 000000000..6c8a83563 --- /dev/null +++ b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/RingMacro.scala @@ -0,0 +1,86 @@ +package com.twitter.algebird.macros + +import com.twitter.algebird.* + +import scala.compiletime.summonInline +import scala.deriving.Mirror +import scala.quoted.Expr +import scala.quoted.Quotes +import scala.quoted.Type +import scala.runtime.Tuples +object RingMacro: + + def summonAll[T: Type](using q: Quotes): List[Expr[Ring[?]]] = + Type.of[T] match + case '[tpe *: tpes] => + Expr.summon[Ring[tpe]] match + case None => derivedRing[tpe] :: summonAll[tpes] + case Some(inst) => inst :: summonAll[tpes] + case '[EmptyTuple] => Nil + + def derivedRing[T: Type](using q: Quotes): Expr[Ring[T]] = + import q.reflect.* + val ev: Expr[Mirror.Of[T]] = Expr.summon[Mirror.Of[T]] match + case None => + val tname = TypeRepr.of[T].typeSymbol.name + report.errorAndAbort(s"unable to find or derive Ring instance for ${tname}") + case Some(expr) => expr + + ev match + case '{ $m: Mirror.ProductOf[T] { type MirroredElemTypes = elementTypes } } => + val insts = summonAll[elementTypes] + val instsAsExpr = Expr.ofTupleFromSeq(insts) + + '{ + new Ring[T]: + override def zero: T = + val t = MonoidMacro.applyZeros(${ instsAsExpr }).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + override def plus(x: T, y: T): T = + val args = Tuple + .fromProduct(x.asInstanceOf[Product]) + .zip(Tuple.fromProduct(y.asInstanceOf[Product])) + .zip(${ instsAsExpr }) + .asInstanceOf[Tuple] + val t = SemigroupMacro.applyPlus(args).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + override def minus(x: T, y: T): T = + val args = Tuple + .fromProduct(x.asInstanceOf[Product]) + .zip(Tuple.fromProduct(y.asInstanceOf[Product])) + .zip(${ instsAsExpr }) + .asInstanceOf[Tuple] + val t = GroupMacro.applyMinus(args).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + override def negate(x: T): T = + val args = Tuple.fromProduct(x.asInstanceOf[Product]).zip(${ instsAsExpr }).asInstanceOf[Tuple] + val t = GroupMacro.applyNegates(args).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + override def one: T = + val t = applyOnes(${ instsAsExpr }).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + override def times(x: T, y: T): T = + val args = Tuple + .fromProduct(x.asInstanceOf[Product]) + .zip(Tuple.fromProduct(y.asInstanceOf[Product])) + .zip(${ instsAsExpr }) + .asInstanceOf[Tuple] + val t = applyTimes(args).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + + } + + case '{ $m: Mirror.SumOf[T] { type MirroredElemTypes = elementTypes } } => + val tname = TypeRepr.of[T].typeSymbol.name + report.errorAndAbort(s"unable to derive Ring for ${tname}") + private[macros] def applyOnes(a: Tuple): Tuple = + a match + case EmptyTuple => EmptyTuple + case inst *: ts => + inst.asInstanceOf[Ring[Any]].one *: applyOnes(ts) + private[macros] def applyTimes(a: Tuple): Tuple = + a match + case EmptyTuple => EmptyTuple + case (((x, y), inst): ((?, ?), Ring[?])) *: ts => + inst.asInstanceOf[Ring[Any]].times(x, y) *: applyTimes(ts) + case _ => ??? diff --git a/algebird-core/src/main/scala-3/com/twitter/algebird/macros/Roller.scala b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/Roller.scala new file mode 100644 index 000000000..1c852752f --- /dev/null +++ b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/Roller.scala @@ -0,0 +1,91 @@ +package com.twitter.algebird.macros +import scala.compiletime.summonInline +import scala.deriving.Mirror +import scala.quoted.Expr +import scala.quoted.Quotes +import scala.quoted.Type +import scala.runtime.Tuples + +/** + * Given a TupleN, produces a sequence of (N + 1) tuples each of arity N such that, for all k from 0 to N, + * there is a tuple with k Somes followed by (N - k) Nones. + * + * This is useful for comparing some metric across multiple layers of some hierarchy. For example, suppose we + * have some climate data represented as case class Data(continent: String, country: String, city: String, + * temperature: Double) and we want to know the average temperatures of + * - each continent + * - each (continent, country) pair + * - each (continent, country, city) triple + * + * Here we desire the (continent, country) and (continent, country, city) pair because, for example, if we + * grouped by city instead of by (continent, country, city), we would accidentally combine the results for + * Paris, Texas and Paris, France. + * + * Then we could do > import com.twitter.algebird.macros.Roller.roller > val data: List[Data] > val + * averageTemps: Map[(Option[String], Option[String], Option[String]), Double] = > data.flatMap { d => + * roller((d.continent, d.country, d.city)).map((_, d)) } > .groupBy(_._1) > .mapValues { xs => val temps = + * xs.map(_.temperature); temps.sum / temps.length } + */ +trait Roller[I]: + type K + def apply(in: I): TraversableOnce[K] + +object Roller extends MacroHelper: + implicit inline def roller[T]: Roller[T] = ${ deriveRollerImpl[T] } + inline def derived[T]: Roller[T] = ${ deriveRollerImpl[T] } + def deriveRollerImpl[T: Type](using q: Quotes): Expr[Roller[T]] = + import q.reflect.* + val tname = TypeRepr.of[T].typeSymbol.name + val ev: Expr[Mirror.Of[T]] = Expr.summon[Mirror.Of[T]] match + case None => + report.errorAndAbort(s"unable to derive Roller instance for ${tname}") + case Some(expr) => expr + ev match + case '{ + $m: Mirror.ProductOf[T] { type MirroredElemTypes = elementTypes } + } => + val caseclassFields = TypeRepr.of[T].typeSymbol.caseFields + if caseclassFields.length > 22 then + report.errorAndAbort( + s"Cannot create Roller for $tname because it has more than 22 parameters." + ) + + val tupleNTyped = genTupleNTyped(caseclassFields) + val somes: List[Symbol] = genSomes(caseclassFields) + val idents = Expr.ofList(somes.map(sym => Ident(sym.termRef).asExpr)) + + val arity = Expr(caseclassFields.length) + + def options(i: Expr[Int], index: Expr[Int]): Expr[Seq[Option[Any]]] = + '{ + (1 to ${ arity }).map(index => + if ((1 << { index - 1 }) & ${ i }) == 0 then None + else ${ idents }(index - 1).asInstanceOf[Option[Any]] + ) + } + def blc(e: Expr[T]): Expr[Any] = + Block( + somesStmt[T](somes, caseclassFields, e), + '{ + (0 to ${ arity }).map { i => + val args = (1 to ${ arity }).map { index => + if (index <= i) ${ idents }(index - 1) else None + } + Tuple.fromArray(args.toArray) + } + + }.asTerm + ).asExpr + tupleNTyped.tpe.widen.asType match + case '[t] => + '{ + + new Roller[T]: + type K = t + override def apply(in: T): TraversableOnce[K] = + ${ blc('{ in }) } + .asInstanceOf[TraversableOnce[K]] + } + + case '{ $m: Mirror.SumOf[T] { type MirroredElemTypes = elementTypes } } => + report.errorAndAbort(s"unable to derive Roller for ${tname}") diff --git a/algebird-core/src/main/scala-3/com/twitter/algebird/macros/SemigroupMacro.scala b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/SemigroupMacro.scala new file mode 100644 index 000000000..aa6ad0544 --- /dev/null +++ b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/SemigroupMacro.scala @@ -0,0 +1,55 @@ +package com.twitter.algebird.macros + +import com.twitter.algebird.* + +import scala.compiletime.summonInline +import scala.deriving.Mirror +import scala.quoted.Expr +import scala.quoted.Quotes +import scala.quoted.Type +import scala.runtime.Tuples +object SemigroupMacro: + + def summonAll[T: Type](using q: Quotes): List[Expr[Semigroup[?]]] = + Type.of[T] match + case '[tpe *: tpes] => + Expr.summon[Semigroup[tpe]] match + case None => derivedSemigroup[tpe] :: summonAll[tpes] + case Some(inst) => inst :: summonAll[tpes] + case '[EmptyTuple] => Nil + + def derivedSemigroup[T: Type](using q: Quotes): Expr[Semigroup[T]] = + import q.reflect.* + val ev: Expr[Mirror.Of[T]] = Expr.summon[Mirror.Of[T]] match + case None => + val tname = TypeRepr.of[T].typeSymbol.name + report.errorAndAbort(s"unable to find or derive Semigroup instance for ${tname}") + case Some(expr) => expr + + ev match + case '{ $m: Mirror.ProductOf[T] { type MirroredElemTypes = elementTypes } } => + val insts = summonAll[elementTypes] + val instsAsExpr = Expr.ofTupleFromSeq(insts) + + '{ + new Semigroup[T]: + override def plus(x: T, y: T): T = + val args = Tuple + .fromProduct(x.asInstanceOf[Product]) + .zip(Tuple.fromProduct(y.asInstanceOf[Product])) + .zip(${ instsAsExpr }) + .asInstanceOf[Tuple] + val t = applyPlus(args).asInstanceOf[elementTypes] + ${ m }.fromTuple(t) + } + + case '{ $m: Mirror.SumOf[T] { type MirroredElemTypes = elementTypes } } => + val tname = TypeRepr.of[T].typeSymbol.name + report.errorAndAbort(s"unable to derive Semigroup for ${tname}") + + private[macros] def applyPlus(a: Tuple): Tuple = + a match + case EmptyTuple => EmptyTuple + case (((x, y), inst): ((?, ?), Semigroup[?])) *: ts => + inst.asInstanceOf[Semigroup[Any]].plus(x, y) *: applyPlus(ts) + case _ => ??? diff --git a/algebird-core/src/main/scala-3/com/twitter/algebird/macros/caseclass.scala b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/caseclass.scala new file mode 100644 index 000000000..bd8436702 --- /dev/null +++ b/algebird-core/src/main/scala-3/com/twitter/algebird/macros/caseclass.scala @@ -0,0 +1,9 @@ +package com.twitter.algebird.macros + +import com.twitter.algebird.* + +object caseclass: + inline final def semigroup[T]: Semigroup[T] = ${ SemigroupMacro.derivedSemigroup[T] } + inline def monoid[T]: Monoid[T] = ${ MonoidMacro.derivedMonoid[T] } + inline def group[T]: Group[T] = ${ GroupMacro.derivedGroup[T] } + inline def ring[T]: Ring[T] = ${ RingMacro.derivedRing[T] } diff --git a/algebird-core/src/main/scala/com/twitter/algebird/Aggregator.scala b/algebird-core/src/main/scala/com/twitter/algebird/Aggregator.scala index a64ce4033..32eec5a37 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/Aggregator.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/Aggregator.scala @@ -10,9 +10,7 @@ import scala.collection.generic.CanBuildFrom * To create a parallel aggregator that operates on a single input in parallel, use: * GeneratedTupleAggregator.from2((agg1, agg2)) */ -object Aggregator extends java.io.Serializable { - implicit def applicative[I]: Applicative[({ type L[O] = Aggregator[I, _, O] })#L] = - new AggregatorApplicative[I] +object Aggregator extends java.io.Serializable with AggregatorCompat { private val DefaultSeed = 471312384 @@ -20,7 +18,7 @@ object Aggregator extends java.io.Serializable { * This is a trivial aggregator that always returns a single value */ def const[T](t: T): MonoidAggregator[Any, Unit, T] = - prepareMonoid { _: Any => () }.andThenPresent(_ => t) + prepareMonoid((_: Any) => ()).andThenPresent(_ => t) /** * Using Aggregator.prepare,present you can add to this aggregator @@ -172,7 +170,7 @@ object Aggregator extends java.io.Serializable { * How many items satisfy a predicate */ def count[T](pred: T => Boolean): MonoidAggregator[T, Long, Long] = - prepareMonoid { t: T => if (pred(t)) 1L else 0L } + prepareMonoid((t: T) => if (pred(t)) 1L else 0L) /** * Do any items satisfy some predicate @@ -310,7 +308,7 @@ object Aggregator extends java.io.Serializable { * Put everything in a Set. Note, this could fill the memory if the Set is very large. */ def toSet[T]: MonoidAggregator[T, Set[T], Set[T]] = - prepareMonoid { t: T => Set(t) } + prepareMonoid((t: T) => Set(t)) /** * This builds an in-memory Set, and then finally gets the size of that set. This may not be scalable if the @@ -506,42 +504,6 @@ trait Aggregator[-A, B, +C] extends java.io.Serializable { self => } } -/** - * Aggregators are Applicatives, but this hides the middle type. If you need a join that does not hide the - * middle type use join on the trait, or GeneratedTupleAggregator.fromN - */ -class AggregatorApplicative[I] extends Applicative[({ type L[O] = Aggregator[I, _, O] })#L] { - override def map[T, U](mt: Aggregator[I, _, T])(fn: T => U): Aggregator[I, _, U] = - mt.andThenPresent(fn) - override def apply[T](v: T): Aggregator[I, _, T] = - Aggregator.const(v) - override def join[T, U](mt: Aggregator[I, _, T], mu: Aggregator[I, _, U]): Aggregator[I, _, (T, U)] = - mt.join(mu) - override def join[T1, T2, T3]( - m1: Aggregator[I, _, T1], - m2: Aggregator[I, _, T2], - m3: Aggregator[I, _, T3] - ): Aggregator[I, _, (T1, T2, T3)] = - GeneratedTupleAggregator.from3((m1, m2, m3)) - - override def join[T1, T2, T3, T4]( - m1: Aggregator[I, _, T1], - m2: Aggregator[I, _, T2], - m3: Aggregator[I, _, T3], - m4: Aggregator[I, _, T4] - ): Aggregator[I, _, (T1, T2, T3, T4)] = - GeneratedTupleAggregator.from4((m1, m2, m3, m4)) - - override def join[T1, T2, T3, T4, T5]( - m1: Aggregator[I, _, T1], - m2: Aggregator[I, _, T2], - m3: Aggregator[I, _, T3], - m4: Aggregator[I, _, T4], - m5: Aggregator[I, _, T5] - ): Aggregator[I, _, (T1, T2, T3, T4, T5)] = - GeneratedTupleAggregator.from5((m1, m2, m3, m4, m5)) -} - trait MonoidAggregator[-A, B, +C] extends Aggregator[A, B, C] { self => def monoid: Monoid[B] override def semigroup: Monoid[B] = monoid diff --git a/algebird-core/src/main/scala/com/twitter/algebird/AveragedValue.scala b/algebird-core/src/main/scala/com/twitter/algebird/AveragedValue.scala index f9f0bb0d8..b5655c545 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/AveragedValue.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/AveragedValue.scala @@ -112,7 +112,7 @@ object AveragedValue { */ def numericAggregator[N](implicit num: Numeric[N]): MonoidAggregator[N, AveragedValue, Double] = Aggregator - .prepareMonoid { n: N => AveragedValue(num.toDouble(n)) } + .prepareMonoid((n: N) => AveragedValue(num.toDouble(n))) .andThenPresent(_.value) /** diff --git a/algebird-core/src/main/scala/com/twitter/algebird/ExpHist.scala b/algebird-core/src/main/scala/com/twitter/algebird/ExpHist.scala index 3a1ebfe2a..d7b815b02 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/ExpHist.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/ExpHist.scala @@ -182,7 +182,7 @@ object ExpHist { case class Bucket(size: Long, timestamp: Timestamp) object Bucket { - implicit val ord: Ordering[Bucket] = Ordering.by { b: Bucket => (b.timestamp, b.size) } + implicit val ord: Ordering[Bucket] = Ordering.by((b: Bucket) => (b.timestamp, b.size)) } /** @@ -260,7 +260,7 @@ object ExpHist { if (desired.isEmpty) Vector.empty else { val input = buckets.dropWhile(_.size == 0) - val bucketSize +: tail = desired + val bucketSize +: tail = desired: @unchecked val remaining = drop(bucketSize, input) input.head.copy(size = bucketSize) +: rebucket(remaining, tail) } @@ -275,7 +275,7 @@ object ExpHist { * If an element wasn't fully consumed, the remainder will be stuck back onto the head. */ @tailrec private[this] def drop(toDrop: Long, input: Vector[Bucket]): Vector[Bucket] = { - val (b @ Bucket(count, _)) +: tail = input + val (b @ Bucket(count, _)) +: tail = input: @unchecked (toDrop - count) match { case 0 => tail case x if x < 0 => b.copy(size = -x) +: tail diff --git a/algebird-core/src/main/scala/com/twitter/algebird/Fold.scala b/algebird-core/src/main/scala/com/twitter/algebird/Fold.scala index ded32e628..7b03ac47e 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/Fold.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/Fold.scala @@ -173,13 +173,7 @@ final class FoldState[X, -I, +O] private[algebird] (val add: (X, I) => X, val st * recommended that "end" functions not mutate the accumulator in order to support scans (producing a stream * of intermediate outputs by calling "end" at each step). */ -object Fold extends CompatFold { - - /** - * "import Fold.applicative" will bring the Applicative instance into scope. See FoldApplicative. - */ - implicit def applicative[I]: Applicative[Fold[I, *]] = - new FoldApplicative[I] +object Fold extends CompatFold with FoldApplicativeCompat { /** * Turn a common Scala foldLeft into a Fold. The accumulator MUST be immutable and serializable. @@ -334,19 +328,3 @@ object Fold extends CompatFold { case (c, _) => c } } - -/** - * Folds are Applicatives! - */ -class FoldApplicative[I] extends Applicative[Fold[I, *]] { - override def map[T, U](mt: Fold[I, T])(fn: T => U): Fold[I, U] = - mt.map(fn) - override def apply[T](v: T): Fold[I, T] = - Fold.const(v) - override def join[T, U](mt: Fold[I, T], mu: Fold[I, U]): Fold[I, (T, U)] = - mt.join(mu) - override def sequence[T](ms: Seq[Fold[I, T]]): Fold[I, Seq[T]] = - Fold.sequence(ms) - override def joinWith[T, U, V](mt: Fold[I, T], mu: Fold[I, U])(fn: (T, U) => V): Fold[I, V] = - mt.joinWith(mu)(fn) -} diff --git a/algebird-core/src/main/scala/com/twitter/algebird/HashingTrick.scala b/algebird-core/src/main/scala/com/twitter/algebird/HashingTrick.scala index 0d86aa03e..7afa4ed97 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/HashingTrick.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/HashingTrick.scala @@ -27,7 +27,7 @@ class HashingTrickMonoid[V: Group](bits: Int, seed: Int = 123456) extends Monoid Monoid.plus(left, right) def init[K](kv: (K, V))(implicit ev: K => Array[Byte]): AdaptiveVector[V] = { - val (long1, long2) = hash(kv._1) + val (long1, long2): (Long, Long) = hash(ev(kv._1)) val index = (long1 & bitMask).toInt val isNegative = (long2 & 1) == 1 diff --git a/algebird-core/src/main/scala/com/twitter/algebird/HyperLogLog.scala b/algebird-core/src/main/scala/com/twitter/algebird/HyperLogLog.scala index 8d2188d16..d0ee201d1 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/HyperLogLog.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/HyperLogLog.scala @@ -556,7 +556,7 @@ class HyperLogLogMonoid(val bits: Int) extends Monoid[HLL] with BoundedSemilatti val size: Int = 1 << bits @deprecated("Use toHLL", since = "0.10.0 / 2015-05") - def apply[T](t: T)(implicit ev: T => Array[Byte]): HLL = create(t) + def apply[T](t: T)(implicit ev: T => Array[Byte]): HLL = create(ev(t)) override val zero: HLL = SparseHLL(bits, Monoid.zero[Map[Int, Max[Byte]]]) @@ -602,7 +602,7 @@ class HyperLogLogMonoid(val bits: Int) extends Monoid[HLL] with BoundedSemilatti @deprecated("Use toHLL", since = "0.10.0 / 2015-05") def batchCreate[T](instances: Iterable[T])(implicit ev: T => Array[Byte]): HLL = { val allMaxRhow = instances - .map(x => jRhoW(hash(x), bits)) + .map(x => jRhoW(hash(ev(x)), bits)) .groupBy { case (j, _) => j } .map { case (j, iter) => (j, Max(iter.maxBy(_._2)._2)) } if (allMaxRhow.size * 16 <= size) { diff --git a/algebird-core/src/main/scala/com/twitter/algebird/InvariantAlgebras.scala b/algebird-core/src/main/scala/com/twitter/algebird/InvariantAlgebras.scala index 5f517e82e..f9ca2acc5 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/InvariantAlgebras.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/InvariantAlgebras.scala @@ -1,28 +1,5 @@ package com.twitter.algebird -class InvariantSemigroup[T, U](val forward: T => U, val reverse: U => T)(implicit val semigroup: Semigroup[T]) - extends Semigroup[U] { - override def plus(l: U, r: U): U = - forward(semigroup.plus(reverse(l), reverse(r))) - override def sumOption(iter: TraversableOnce[U]): Option[U] = - semigroup.sumOption(iter.map(reverse)).map(forward) - - /* - * Note these work for the subclasses since in those cases semigroup - * will be the appropriate algebra. - */ - override val hashCode: Int = (forward, reverse, semigroup).hashCode - override def equals(that: Any): Boolean = - that match { - case r: InvariantSemigroup[_, _] => - (hashCode == r.hashCode) && - (forward == r.forward) && - (reverse == r.reverse) && - (semigroup == r.semigroup) - case _ => false - } -} - class InvariantMonoid[T, U](forward: T => U, reverse: U => T)(implicit val monoid: Monoid[T]) extends InvariantSemigroup[T, U](forward, reverse) with Monoid[U] { diff --git a/algebird-core/src/main/scala/com/twitter/algebird/JavaMonoids.scala b/algebird-core/src/main/scala/com/twitter/algebird/JavaMonoids.scala index 5fc8d6dc4..dd49f3621 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/JavaMonoids.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/JavaMonoids.scala @@ -23,9 +23,7 @@ import java.lang.{ Long => JLong, Short => JShort } -import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} - -import scala.jdk.CollectionConverters._ +import java.util.{ArrayList => JArrayList, List => JList} object JIntRing extends Ring[JInt] { override val zero: JInt = JInt.valueOf(0) @@ -97,51 +95,3 @@ class JListMonoid[T] extends Monoid[JList[T]] { res } } - -/** - * Since maps are mutable, this always makes a full copy. Prefer scala immutable maps if you use scala - * immutable maps, this operation is much faster TODO extend this to Group, Ring - */ -class JMapMonoid[K, V: Semigroup] extends Monoid[JMap[K, V]] { - override lazy val zero: JHashMap[K, V] = new JHashMap[K, V](0) - - val nonZero: (V => Boolean) = implicitly[Semigroup[V]] match { - case mon: Monoid[_] => mon.isNonZero(_) - case _ => _ => true - } - - override def isNonZero(x: JMap[K, V]): Boolean = - !x.isEmpty && (implicitly[Semigroup[V]] match { - case mon: Monoid[_] => - x.values.asScala.exists(v => mon.isNonZero(v)) - case _ => true - }) - override def plus(x: JMap[K, V], y: JMap[K, V]): JHashMap[K, V] = { - val (big, small, bigOnLeft) = - if (x.size > y.size) { - (x, y, true) - } else { - (y, x, false) - } - val vsemi = implicitly[Semigroup[V]] - val result = new JHashMap[K, V](big.size + small.size) - result.putAll(big) - small.entrySet.asScala.foreach { kv => - val smallK = kv.getKey - val smallV = kv.getValue - if (big.containsKey(smallK)) { - val bigV = big.get(smallK) - val newV = - if (bigOnLeft) vsemi.plus(bigV, smallV) else vsemi.plus(smallV, bigV) - if (nonZero(newV)) - result.put(smallK, newV) - else - result.remove(smallK) - } else { - // No need to explicitly add with zero on V, just put in the small value - result.put(smallK, smallV) - } - } - result - } -} diff --git a/algebird-core/src/main/scala/com/twitter/algebird/MapAlgebra.scala b/algebird-core/src/main/scala/com/twitter/algebird/MapAlgebra.scala index 55a9f8e54..357bcd1f1 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/MapAlgebra.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/MapAlgebra.scala @@ -27,76 +27,6 @@ trait MapOperations[K, V, M <: ScMap[K, V]] { def fromMutable(mut: MMap[K, V]): M } -abstract class GenericMapMonoid[K, V, M <: ScMap[K, V]](implicit val semigroup: Semigroup[V]) - extends Monoid[M] - with MapOperations[K, V, M] { - - val nonZero: (V => Boolean) = semigroup match { - case mon: Monoid[_] => mon.isNonZero(_) - case _ => _ => true - } - - override def isNonZero(x: M): Boolean = - !x.isEmpty && (semigroup match { - case mon: Monoid[_] => - x.valuesIterator.exists(v => mon.isNonZero(v)) - case _ => true - }) - - override def plus(x: M, y: M): M = { - // Scala maps can reuse internal structure, so don't copy just add into the bigger one: - // This really saves computation when adding lots of small maps into big ones (common) - val (big, small, bigOnLeft) = - if (x.size > y.size) { - (x, y, true) - } else { - (y, x, false) - } - small match { - // Mutable maps create new copies of the underlying data on add so don't use the - // handleImmutable method. - // Cannot have a None so 'get' is safe here. - case _: MMap[_, _] => sumOption(Seq(big, small)).get - case _ => handleImmutable(big, small, bigOnLeft) - } - } - - private def handleImmutable(big: M, small: M, bigOnLeft: Boolean) = - small.foldLeft(big) { (oldMap, kv) => - val newV = big - .get(kv._1) - .map { bigV => - if (bigOnLeft) - semigroup.plus(bigV, kv._2) - else - semigroup.plus(kv._2, bigV) - } - .getOrElse(kv._2) - if (nonZero(newV)) - add(oldMap, kv._1 -> newV) - else - remove(oldMap, kv._1) - } - override def sumOption(items: TraversableOnce[M]): Option[M] = - if (items.iterator.isEmpty) None - else { - val mutable = MMap[K, V]() - items.iterator.foreach { m => - m.foreach { case (k, v) => - val oldVOpt = mutable.get(k) - // sorry for the micro optimization here: avoiding a closure - val newV = - if (oldVOpt.isEmpty) v else Semigroup.plus(oldVOpt.get, v) - if (nonZero(newV)) - mutable.update(k, newV) - else - mutable.remove(k) - } - } - Some(fromMutable(mutable)) - } -} - class MapMonoid[K, V](implicit semigroup: Semigroup[V]) extends GenericMapMonoid[K, V, Map[K, V]] { override lazy val zero: Map[K, V] = Map[K, V]() override def add(oldMap: Map[K, V], kv: (K, V)): Map[K, V] = oldMap + kv @@ -229,7 +159,7 @@ object MapAlgebra { // Consider this as edges from k -> v, produce a Map[K,Set[V]] def toGraph[K, V](pairs: TraversableOnce[(K, V)]): Map[K, Set[V]] = - Monoid.sum(pairs.map { case (k, v) => Map(k -> Set(v)) }) + Monoid.sum(pairs.iterator.map { case (k, v) => Map(k -> Set(v)) }) /** join the keys of two maps (similar to outer-join in a DB) */ def join[K, V, W](map1: Map[K, V], map2: Map[K, W]): Map[K, (Option[V], Option[W])] = diff --git a/algebird-core/src/main/scala/com/twitter/algebird/Metric.scala b/algebird-core/src/main/scala/com/twitter/algebird/Metric.scala index e5c6df39b..fc4dd10e8 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/Metric.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/Metric.scala @@ -73,7 +73,7 @@ object Metric { def minkowskiMap[K, V: Monoid: Metric](p: Double): Metric[Map[K, V]] = Metric.from { (a: Map[K, V], b: Map[K, V]) => - val outP = (a.keySet ++ b.keySet).map { key: K => + val outP = (a.keySet ++ b.keySet).map { (key: K) => val v1 = a.getOrElse(key, Monoid.zero[V]) val v2 = b.getOrElse(key, Monoid.zero[V]) math.pow(implicitly[Metric[V]].apply(v1, v2), p) diff --git a/algebird-core/src/main/scala/com/twitter/algebird/MomentsGroup.scala b/algebird-core/src/main/scala/com/twitter/algebird/MomentsGroup.scala index 74eb5a428..e1b93da68 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/MomentsGroup.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/MomentsGroup.scala @@ -248,7 +248,7 @@ object Moments { val fold: Fold[Double, Moments] = momentsMonoid.zero.fold def numericAggregator[N](implicit num: Numeric[N]): MonoidAggregator[N, Moments, Moments] = - Aggregator.prepareMonoid { n: N => Moments(num.toDouble(n)) } + Aggregator.prepareMonoid((n: N) => Moments(num.toDouble(n))) /** * Create a Moments object given a single value. This is useful for initializing moment calculations at the diff --git a/algebird-core/src/main/scala/com/twitter/algebird/Monad.scala b/algebird-core/src/main/scala/com/twitter/algebird/Monad.scala index de8c31a71..0b891ed89 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/Monad.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/Monad.scala @@ -57,7 +57,7 @@ object Monad { if (xs.isEmpty) monad.apply(acc) else - monad.flatMap(fn(acc, xs.head)) { t: T => foldM(t, xs.tail)(fn) } + monad.flatMap(fn(acc, xs.head))((t: T) => foldM(t, xs.tail)(fn)) // Some instances of the Monad typeclass (case for a macro): implicit val list: Monad[List] = new Monad[List] { diff --git a/algebird-core/src/main/scala/com/twitter/algebird/Preparer.scala b/algebird-core/src/main/scala/com/twitter/algebird/Preparer.scala index a10d6d8a8..71399afcd 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/Preparer.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/Preparer.scala @@ -187,10 +187,10 @@ trait FlatMapPreparer[A, T] extends Preparer[A, T] { def prepareFn: A => TraversableOnce[T] def map[U](fn: T => U): FlatMapPreparer[A, U] = - FlatMapPreparer { a: A => prepareFn(a).map(fn) } + FlatMapPreparer((a: A) => prepareFn(a).map(fn)) override def flatMap[U](fn: T => TraversableOnce[U]): FlatMapPreparer[A, U] = - FlatMapPreparer { a: A => prepareFn(a).flatMap(fn) } + FlatMapPreparer((a: A) => prepareFn(a).flatMap(fn)) override def monoidAggregate[B, C](aggregator: MonoidAggregator[T, B, C]): MonoidAggregator[A, B, C] = aggregator.sumBefore.composePrepare(prepareFn) @@ -242,10 +242,10 @@ object FlatMapPreparer { override val prepareFn: TraversableOnce[A] => TraversableOnce[A] = (a: TraversableOnce[A]) => a override def map[U](fn: A => U): FlatMapPreparer[TraversableOnce[A], U] = - FlatMapPreparer { a: TraversableOnce[A] => a.map(fn) } + FlatMapPreparer((a: TraversableOnce[A]) => a.map(fn)) override def flatMap[U](fn: A => TraversableOnce[U]): FlatMapPreparer[TraversableOnce[A], U] = - FlatMapPreparer { a: TraversableOnce[A] => a.flatMap(fn) } + FlatMapPreparer((a: TraversableOnce[A]) => a.flatMap(fn)) override def monoidAggregate[B, C]( aggregator: MonoidAggregator[A, B, C] diff --git a/algebird-core/src/main/scala/com/twitter/algebird/Scan.scala b/algebird-core/src/main/scala/com/twitter/algebird/Scan.scala index d1d10ced7..7c2d23303 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/Scan.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/Scan.scala @@ -2,7 +2,7 @@ package com.twitter.algebird import scala.collection.compat._ -object Scan { +object Scan extends ScanApplicativeCompat { /** * Most consumers of Scan don't care about the type of the type State type variable. But for those that do, @@ -13,8 +13,6 @@ object Scan { */ type Aux[-I, S, +O] = Scan[I, O] { type State = S } - implicit def applicative[I]: Applicative[Scan[I, *]] = new ScanApplicative[I] - def from[I, S, O](initState: S)(presentAndNextStateFn: (I, S) => (O, S)): Aux[I, S, O] = new Scan[I, O] { override type State = S @@ -320,14 +318,3 @@ sealed abstract class Scan[-I, +O] extends Serializable { } } - -class ScanApplicative[I] extends Applicative[Scan[I, *]] { - override def map[T, U](mt: Scan[I, T])(fn: T => U): Scan[I, U] = - mt.andThenPresent(fn) - - override def apply[T](v: T): Scan[I, T] = - Scan.const(v) - - override def join[T, U](mt: Scan[I, T], mu: Scan[I, U]): Scan[I, (T, U)] = - mt.join(mu) -} diff --git a/algebird-core/src/main/scala/com/twitter/algebird/SummingCache.scala b/algebird-core/src/main/scala/com/twitter/algebird/SummingCache.scala index 01f68d141..d7f6f356b 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/SummingCache.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/SummingCache.scala @@ -64,16 +64,18 @@ class SummingCache[K, V](capacity: Int)(implicit sgv: Semigroup[V]) extends Stat protected var lastEvicted: Map[K, V] = Map.empty[K, V] // TODO fancier caches will give better performance: - protected lazy val cache: MMap[K, V] = + protected lazy val cache: MMap[K, V] = { + val cap = capacity (new JLinkedHashMap[K, V](capacity + 1, 0.75f, true) { override protected def removeEldestEntry(eldest: JMap.Entry[K, V]): Boolean = - if (super.size > capacity) { + if (super.size > cap) { lastEvicted += (eldest.getKey -> eldest.getValue) true } else { false } }).asScala + } } object SummingWithHitsCache { diff --git a/algebird-core/src/main/scala/com/twitter/algebird/immutable/BitSet.scala b/algebird-core/src/main/scala/com/twitter/algebird/immutable/BitSet.scala index d58a6c9ab..dfe9e6151 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/immutable/BitSet.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/immutable/BitSet.scala @@ -573,7 +573,7 @@ object BitSet { BitSet.adoptedUnion(this, rhs) } else { // height == rhs.height, so we know rhs is a Branch. - val Branch(_, _, rcs) = rhs + val Branch(_, _, rcs) = rhs: @unchecked val cs = new Array[BitSet](32) var i = 0 while (i < 32) { @@ -605,7 +605,7 @@ object BitSet { Empty } else { // height == rhs.height, so we know rhs is a Branch. - val Branch(_, _, rcs) = rhs + val Branch(_, _, rcs) = rhs: @unchecked val cs = new Array[BitSet](32) var i = 0 var nonEmpty = false @@ -643,7 +643,7 @@ object BitSet { false } else { // height == rhs.height, so we know rhs is a Branch. - val Branch(_, _, rcs) = rhs + val Branch(_, _, rcs) = rhs: @unchecked var i = 0 while (i < 32) { val x = children(i) @@ -688,7 +688,7 @@ object BitSet { this | rhs } else { // height == rhs.height, so we know rhs is a Branch. - val Branch(_, _, rcs) = rhs + val Branch(_, _, rcs) = rhs: @unchecked val cs = new Array[BitSet](32) var i = 0 while (i < 32) { @@ -805,7 +805,7 @@ object BitSet { throw InternalError("branch misaligned") } else { // height == rhs.height, so we know rhs is a Branch. - val Branch(_, _, rcs) = rhs + val Branch(_, _, rcs) = rhs: @unchecked var i = 0 while (i < 32) { val x = children(i) diff --git a/algebird-core/src/main/scala/com/twitter/algebird/immutable/BloomFilter.scala b/algebird-core/src/main/scala/com/twitter/algebird/immutable/BloomFilter.scala index 71a861075..77dc32d15 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/immutable/BloomFilter.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/immutable/BloomFilter.scala @@ -259,7 +259,7 @@ final case class BloomFilter[A](numHashes: Int, width: Int)(implicit val hash: H } case class Item(item: A) extends Hash { - override val numBits: Int = numHashes + override val numBits: Int = this.numHashes override def toBitSet: BitSet = BitSet(hashToArray(item)) @@ -272,7 +272,7 @@ final case class BloomFilter[A](numHashes: Int, width: Int)(implicit val hash: H override def +(other: A): Hash = { val bs = BitSet.newEmpty(0) - val hash = new Array[Int](numHashes) + val hash = new Array[Int](this.numHashes) hashToArray(item, hash) bs.mutableAdd(hash) @@ -336,7 +336,7 @@ final case class BloomFilter[A](numHashes: Int, width: Int)(implicit val hash: H // use an approximation width of 0.05 override def size: Approximate[Long] = - BloomFilter.sizeEstimate(numBits, numHashes, width, 0.05) + BloomFilter.sizeEstimate(this.numBits, this.numHashes, this.width, 0.05) } implicit val monoid: Monoid[Hash] with BoundedSemilattice[Hash] = @@ -350,7 +350,7 @@ final case class BloomFilter[A](numHashes: Int, width: Int)(implicit val hash: H override def plus(left: Hash, right: Hash): Hash = left ++ right override def sum(t: TraversableOnce[Hash]): Hash = - if (t.iterator.isEmpty) empty + if (t.iterator.isEmpty) this.empty else { val iter = t.iterator var bs = BitSet.newEmpty(0) @@ -402,7 +402,7 @@ final case class BloomFilter[A](numHashes: Int, width: Int)(implicit val hash: H /** * Create a bloom filter with multiple items from an iterator */ - def create(data: Iterator[A]): Hash = monoid.sum(data.map(Item)) + def create(data: Iterator[A]): Hash = monoid.sum(data.map(Item.apply)) val empty: Hash = Empty diff --git a/algebird-core/src/main/scala/com/twitter/algebird/matrix/SparseColumnMatrix.scala b/algebird-core/src/main/scala/com/twitter/algebird/matrix/SparseColumnMatrix.scala index b1f95b21a..8e57d064f 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/matrix/SparseColumnMatrix.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/matrix/SparseColumnMatrix.scala @@ -49,7 +49,7 @@ case class SparseColumnMatrix[V: Monoid](rowsByColumns: IndexedSeq[AdaptiveVecto while (row < rows) { val iter = rowsByColumns(row).denseIterator while (iter.hasNext) { - val (col, value) = iter.next() + val (col, value): (Int, V) = iter.next() val indx = row * lcols + col buffer(indx) = valueMonoid.plus(buffer(indx), value) } diff --git a/algebird-core/src/main/scala/com/twitter/algebird/monad/Trampoline.scala b/algebird-core/src/main/scala/com/twitter/algebird/monad/Trampoline.scala index 07e296c9b..b24508531 100644 --- a/algebird-core/src/main/scala/com/twitter/algebird/monad/Trampoline.scala +++ b/algebird-core/src/main/scala/com/twitter/algebird/monad/Trampoline.scala @@ -66,7 +66,8 @@ object Trampoline { case next :: tail => loop(next(a), tail) case Nil => a } - case FlatMapped(item, fn) => loop(item, fn :: stack) + case FlatMapped(item, fn) => + loop(item, fn.asInstanceOf[Any => com.twitter.algebird.monad.Trampoline[Any]] :: stack) } // Sorry for the cast, but it is tough to get the types right without a lot of wrapping loop(tramp, Nil).asInstanceOf[A] diff --git a/algebird-core/src/test/java/com/twitter/algebird/javaapi/TestRegistry.java b/algebird-core/src/test/java-2/com/twitter/algebird/javaapi/TestRegistry.java similarity index 100% rename from algebird-core/src/test/java/com/twitter/algebird/javaapi/TestRegistry.java rename to algebird-core/src/test/java-2/com/twitter/algebird/javaapi/TestRegistry.java diff --git a/algebird-test/src/main/scala-2.11/com/twitter/algebird/BasePropertiesCompat.scala b/algebird-test/src/main/scala-2.11/com/twitter/algebird/BasePropertiesCompat.scala new file mode 100644 index 000000000..9a3c2a7b2 --- /dev/null +++ b/algebird-test/src/main/scala-2.11/com/twitter/algebird/BasePropertiesCompat.scala @@ -0,0 +1,9 @@ +package com.twitter.algebird + +private[algebird] trait BasePropertiesCompat { + def isNonZero[V: Semigroup](v: V): Boolean = + implicitly[Semigroup[V]] match { + case mon: Monoid[_] => mon.isNonZero(v) + case _ => true + } +} diff --git a/algebird-test/src/main/scala-2.12+/com/twitter/algebird/BasePropertiesCompat.scala b/algebird-test/src/main/scala-2.12+/com/twitter/algebird/BasePropertiesCompat.scala new file mode 100644 index 000000000..1e4c616e5 --- /dev/null +++ b/algebird-test/src/main/scala-2.12+/com/twitter/algebird/BasePropertiesCompat.scala @@ -0,0 +1,9 @@ +package com.twitter.algebird + +private[algebird] trait BasePropertiesCompat { + def isNonZero[V: Semigroup](v: V): Boolean = + implicitly[Semigroup[V]] match { + case mon: Monoid[?] => mon.isNonZero(v) + case _ => true + } +} diff --git a/algebird-test/src/main/scala/com/twitter/algebird/macros/ArbitraryCaseClassMacro.scala b/algebird-test/src/main/scala-2/com/twitter/algebird/macros/ArbitraryCaseClassMacro.scala similarity index 100% rename from algebird-test/src/main/scala/com/twitter/algebird/macros/ArbitraryCaseClassMacro.scala rename to algebird-test/src/main/scala-2/com/twitter/algebird/macros/ArbitraryCaseClassMacro.scala diff --git a/algebird-test/src/main/scala-3/com/twitter/algebird/macros/ArbitraryCaseClassMacro.scala b/algebird-test/src/main/scala-3/com/twitter/algebird/macros/ArbitraryCaseClassMacro.scala new file mode 100644 index 000000000..50412ae77 --- /dev/null +++ b/algebird-test/src/main/scala-3/com/twitter/algebird/macros/ArbitraryCaseClassMacro.scala @@ -0,0 +1,71 @@ +package com.twitter.algebird.macros + +import com.twitter.algebird.* + +import scala.compiletime.summonInline +import scala.deriving.Mirror +import scala.quoted.Expr +import scala.quoted.Quotes +import scala.quoted.Type +import scala.runtime.Tuples +import org.scalacheck.{Arbitrary, Gen} + +object ArbitraryCaseClassMacro: + inline def gen[T]: Gen[T] = ${ caseClassGen[T] } + inline def arbitrary[T]: Arbitrary[T] = ${ caseClassArbitrary[T] } + def summonAll[T: Type](using q: Quotes): List[Expr[Gen[?]]] = + Type.of[T] match + case '[tpe *: tpes] => + Expr.summon[Arbitrary[tpe]] match + case None => caseClassGen[tpe] :: summonAll[tpes] + case Some(inst) => '{ ${ inst }.arbitrary } :: summonAll[tpes] + case '[EmptyTuple] => Nil + def caseClassArbitrary[T: Type](using q: Quotes): Expr[Arbitrary[T]] = + import q.reflect.* + val gen = caseClassGen[T] + '{ + Arbitrary[T]($gen) + } + + def caseClassGen[T: Type](using q: Quotes): Expr[Gen[T]] = + import q.reflect.* + val ev: Expr[Mirror.Of[T]] = Expr.summon[Mirror.Of[T]] match + case None => + val tname = TypeRepr.of[T].typeSymbol.name + report.errorAndAbort( + s"unable to find or derive Arbitrary instance for ${tname}" + ) + case Some(expr) => expr + + ev match + case '{ + $m: Mirror.ProductOf[T] { type MirroredElemTypes = elementTypes } + } => + val insts: List[Expr[Gen[?]]] = summonAll[elementTypes].reverse + val last :: leading = insts: @unchecked + + val expr = + leading.foldLeft[Seq[Expr[?]] => Expr[Gen[T]]] { args => + val argsExpr = Expr.ofSeq(args) + '{ + ${ last }.map { v => + ${ m }.fromTuple( + Tuple + .fromArray((${ argsExpr } :+ v).toArray) + .asInstanceOf[elementTypes] + ) + } + } + } { case (acc, gen) => + args => + '{ + ${ gen }.flatMap { v => + ${ acc(args :+ '{ v }) } + } + } + } + expr(Nil) + + case '{ $m: Mirror.SumOf[T] { type MirroredElemTypes = elementTypes } } => + val tname = TypeRepr.of[T].typeSymbol.name + report.errorAndAbort(s"unable to derive Arbitrary for ${tname}") diff --git a/algebird-test/src/main/scala/com/twitter/algebird/ApproximateProperty.scala b/algebird-test/src/main/scala/com/twitter/algebird/ApproximateProperty.scala index 736623c4b..03cecc7a7 100644 --- a/algebird-test/src/main/scala/com/twitter/algebird/ApproximateProperty.scala +++ b/algebird-test/src/main/scala/com/twitter/algebird/ApproximateProperty.scala @@ -58,7 +58,7 @@ object ApproximateProperty { } def toProp(a: ApproximateProperty, objectReps: Int, inputReps: Int, falsePositiveRate: Double): Prop = - Prop { _: Gen.Parameters => + Prop { (_: Gen.Parameters) => require(0 <= falsePositiveRate && falsePositiveRate <= 1) val list = successesAndProbabilities(a, objectReps, inputReps) diff --git a/algebird-test/src/main/scala/com/twitter/algebird/BaseProperties.scala b/algebird-test/src/main/scala/com/twitter/algebird/BaseProperties.scala index 4b81738d5..b5578987e 100644 --- a/algebird-test/src/main/scala/com/twitter/algebird/BaseProperties.scala +++ b/algebird-test/src/main/scala/com/twitter/algebird/BaseProperties.scala @@ -24,7 +24,7 @@ import scala.math.Equiv /** * Base properties useful for all tests using Algebird's typeclasses. */ -object BaseProperties extends MetricProperties { +object BaseProperties extends MetricProperties with BasePropertiesCompat { /** * We generate a restricted set of BigDecimals for our tests because if we use the full range then the way @@ -170,12 +170,6 @@ object BaseProperties extends MetricProperties { override def apply[T](m: M[T], n: M[T]): Boolean = m == n } - def isNonZero[V: Semigroup](v: V): Boolean = - implicitly[Semigroup[V]] match { - case mon: Monoid[_] => mon.isNonZero(v) - case _ => true - } - def isAssociativeDifferentTypes[T: Semigroup: Equiv, U <: T: Arbitrary]: Prop = "isAssociativeEq" |: forAll { (a: U, b: U, c: U) => val semi = implicitly[Semigroup[T]] diff --git a/algebird-test/src/main/scala/com/twitter/algebird/scalacheck/Gen.scala b/algebird-test/src/main/scala/com/twitter/algebird/scalacheck/Gen.scala index f9cbd0d11..6a401feba 100644 --- a/algebird-test/src/main/scala/com/twitter/algebird/scalacheck/Gen.scala +++ b/algebird-test/src/main/scala/com/twitter/algebird/scalacheck/Gen.scala @@ -47,9 +47,9 @@ object gen extends ExpHistGen with IntervalGen { for { m0 <- choose(1L, Int.MaxValue.toLong) m1 <- choose(-1e50, 1e50) - m2 <- choose(0, 1e50) + m2 <- choose[Double](0, 1e50) m3 <- choose(-1e10, 1e50) - m4 <- choose(0, 1e50) + m4 <- choose[Double](0, 1e50) } yield new Moments(m0, m1, m2, m3, m4) private val genLongString: Gen[String] = for { @@ -91,8 +91,8 @@ object gen extends ExpHistGen with IntervalGen { val genRandom: Gen[Correlation] = for { c2 <- choose(-1e10, 1e10) - m2x <- choose(0, 1e10) - m2y <- choose(0, 1e10) + m2x <- choose[Double](0, 1e10) + m2y <- choose[Double](0, 1e10) m1x <- choose(-1e10, 1e10) m1y <- choose(-1e10, 1e10) m0 <- choose(-1e10, 1e10) diff --git a/algebird-test/src/test/scala-2.11/com/twitter/algebird/AggregatorLawsCompat.scala b/algebird-test/src/test/scala-2.11/com/twitter/algebird/AggregatorLawsCompat.scala new file mode 100644 index 000000000..710446663 --- /dev/null +++ b/algebird-test/src/test/scala-2.11/com/twitter/algebird/AggregatorLawsCompat.scala @@ -0,0 +1,28 @@ +package com.twitter.algebird + +import org.scalacheck.Arbitrary +import org.scalacheck.Prop._ +import org.scalacheck.Prop +trait AggregatorLawsCompat { self: CheckProperties => + implicit def aggregator[A, B, C](implicit + prepare: Arbitrary[A => B], + sg: Semigroup[B], + present: Arbitrary[B => C] + ): Arbitrary[Aggregator[A, B, C]] = Arbitrary { + for { + pp <- prepare.arbitrary + ps <- present.arbitrary + } yield new Aggregator[A, B, C] { + def prepare(a: A) = pp(a) + def semigroup = sg + def present(b: B) = ps(b) + } + } + property("Applicative composing two Aggregators is correct") { + forAll { (in: List[Int], ag1: Aggregator[Int, Set[Int], Int], ag2: Aggregator[Int, Unit, String]) => + type AggInt[T] = Aggregator[Int, _, T] + val c = Applicative.join[AggInt, Int, String](ag1, ag2) + in.isEmpty || c(in) == ((ag1(in), ag2(in))) + } + } +} diff --git a/algebird-test/src/test/scala-2.11/com/twitter/algebird/CMSContraMapSpecCompat.scala b/algebird-test/src/test/scala-2.11/com/twitter/algebird/CMSContraMapSpecCompat.scala new file mode 100644 index 000000000..9c46c7f34 --- /dev/null +++ b/algebird-test/src/test/scala-2.11/com/twitter/algebird/CMSContraMapSpecCompat.scala @@ -0,0 +1,37 @@ +package com.twitter.algebird + +import org.scalatestplus.scalacheck.{ScalaCheckDrivenPropertyChecks, ScalaCheckPropertyChecks} +import org.scalacheck.{Arbitrary, Gen} + +import scala.util.Random +import CMSHasherImplicits.CMSHasherBigInt +import org.scalatest.matchers.should.Matchers +import org.scalatest.propspec.AnyPropSpec +import org.scalatest.wordspec.AnyWordSpec + +trait CMSContraMapSpecCompat { self: AnyWordSpec with Matchers with ScalaCheckDrivenPropertyChecks => + "translates CMSHasher[K] into CMSHasher[L], given a function f: L => K" in { + // Given a "source" CMSHasher[K] + val sourceHasher: CMSHasher[String] = CMSHasher.CMSHasherString + // and a translation function from an unsupported type L (here: Seq[Byte]) to K + def f(bytes: Seq[Byte]): String = new String(bytes.toArray[Byte], "UTF-8") + + // When we run contramap on a CMSHasher[K] supplying f, + // then the result should be a CMSHasher[L]... + val targetHasher: CMSHasher[Seq[Byte]] = + sourceHasher.contramap((d: Seq[Byte]) => f(d)) + targetHasher shouldBe an[CMSHasher[ + _ + ]] // Can't test CMSHasher[Seq[Byte]] specifically because of type erasure. + + // ...and hashing should work correctly (this is only a smoke test). + val a = 4 + val b = 0 + val width = 1234 + val x = Array(113.toByte).toSeq // same as Seq(133.toByte) + val result = targetHasher.hash(a, b, width)(x) + val expected = sourceHasher.hash(a, b, width)("q") + result should be(expected) + result should be(434) + } +} diff --git a/algebird-test/src/test/scala-2.11/com/twitter/algebird/IntervalLawsCompat.scala b/algebird-test/src/test/scala-2.11/com/twitter/algebird/IntervalLawsCompat.scala new file mode 100644 index 000000000..bcc4afb5f --- /dev/null +++ b/algebird-test/src/test/scala-2.11/com/twitter/algebird/IntervalLawsCompat.scala @@ -0,0 +1,38 @@ +package com.twitter.algebird + +import org.scalacheck.Arbitrary +import org.scalacheck.Prop._ +import org.scalacheck.Prop +import org.scalatest.funsuite.AnyFunSuite + +trait IntervalLawsCompat { self: CheckProperties => + import com.twitter.algebird.scalacheck.arbitrary._ + import com.twitter.algebird.Interval.GenIntersection + property("if boundedLeast is none, we are Universe, Upper or isEmpty is true") { + forAll { (a: Interval[Long]) => + a.boundedLeast match { + case Some(_) => true + case None => + a.isEmpty || (a match { + case _: Upper[_] => true + case Universe() => true + case _ => false + }) + } + } + } + property("if boundedGreatest is none, we are Universe, Lower or isEmpty is true") { + forAll { (a: Interval[Long]) => + a.boundedGreatest match { + case Some(_) => true + case None => + a.isEmpty || (a match { + case _: Lower[_] => true + case Universe() => true + case _ => false + }) + } + } + } + +} diff --git a/algebird-test/src/test/scala-2.12+/com/twitter/algebird/AggregatorLawsCompat.scala b/algebird-test/src/test/scala-2.12+/com/twitter/algebird/AggregatorLawsCompat.scala new file mode 100644 index 000000000..bbf351ce2 --- /dev/null +++ b/algebird-test/src/test/scala-2.12+/com/twitter/algebird/AggregatorLawsCompat.scala @@ -0,0 +1,27 @@ +package com.twitter.algebird + +import org.scalacheck.Arbitrary +import org.scalacheck.Prop._ +trait AggregatorLawsCompat { self: CheckProperties => + implicit def aggregator[A, B, C](implicit + prepare: Arbitrary[A => B], + sg: Semigroup[B], + present: Arbitrary[B => C] + ): Arbitrary[Aggregator[A, B, C]] = Arbitrary { + for { + pp <- prepare.arbitrary + ps <- present.arbitrary + } yield new Aggregator[A, B, C] { + def prepare(a: A) = pp(a) + def semigroup = sg + def present(b: B) = ps(b) + } + } + property("Applicative composing two Aggregators is correct") { + forAll { (in: List[Int], ag1: Aggregator[Int, Set[Int], Int], ag2: Aggregator[Int, Unit, String]) => + type AggInt[T] = Aggregator[Int, ?, T] + val c = Applicative.join[AggInt, Int, String](ag1, ag2) + in.isEmpty || c(in) == ((ag1(in), ag2(in))) + } + } +} diff --git a/algebird-test/src/test/scala-2.12+/com/twitter/algebird/CMSContraMapSpecCompat.scala b/algebird-test/src/test/scala-2.12+/com/twitter/algebird/CMSContraMapSpecCompat.scala new file mode 100644 index 000000000..c6b183695 --- /dev/null +++ b/algebird-test/src/test/scala-2.12+/com/twitter/algebird/CMSContraMapSpecCompat.scala @@ -0,0 +1,33 @@ +package com.twitter.algebird + +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks + +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +trait CMSContraMapSpecCompat { self: AnyWordSpec with Matchers with ScalaCheckDrivenPropertyChecks => + "translates CMSHasher[K] into CMSHasher[L], given a function f: L => K" in { + // Given a "source" CMSHasher[K] + val sourceHasher: CMSHasher[String] = CMSHasher.CMSHasherString + // and a translation function from an unsupported type L (here: Seq[Byte]) to K + def f(bytes: Seq[Byte]): String = new String(bytes.toArray[Byte], "UTF-8") + + // When we run contramap on a CMSHasher[K] supplying f, + // then the result should be a CMSHasher[L]... + val targetHasher: CMSHasher[Seq[Byte]] = + sourceHasher.contramap((d: Seq[Byte]) => f(d)) + targetHasher shouldBe an[CMSHasher[ + ? + ]] // Can't test CMSHasher[Seq[Byte]] specifically because of type erasure. + + // ...and hashing should work correctly (this is only a smoke test). + val a = 4 + val b = 0 + val width = 1234 + val x = Array(113.toByte).toSeq // same as Seq(133.toByte) + val result = targetHasher.hash(a, b, width)(x) + val expected = sourceHasher.hash(a, b, width)("q") + result should be(expected) + result should be(434) + } +} diff --git a/algebird-test/src/test/scala-2.12+/com/twitter/algebird/IntervalLawsCompat.scala b/algebird-test/src/test/scala-2.12+/com/twitter/algebird/IntervalLawsCompat.scala new file mode 100644 index 000000000..09f95e660 --- /dev/null +++ b/algebird-test/src/test/scala-2.12+/com/twitter/algebird/IntervalLawsCompat.scala @@ -0,0 +1,34 @@ +package com.twitter.algebird + +import org.scalacheck.Prop._ + +trait IntervalLawsCompat { self: CheckProperties => + import com.twitter.algebird.scalacheck.arbitrary._ + property("if boundedLeast is none, we are Universe, Upper or isEmpty is true") { + forAll { (a: Interval[Long]) => + a.boundedLeast match { + case Some(_) => true + case None => + a.isEmpty || (a match { + case _: Upper[?] => true + case Universe() => true + case _ => false + }) + } + } + } + property("if boundedGreatest is none, we are Universe, Lower or isEmpty is true") { + forAll { (a: Interval[Long]) => + a.boundedGreatest match { + case Some(_) => true + case None => + a.isEmpty || (a match { + case _: Lower[?] => true + case Universe() => true + case _ => false + }) + } + } + } + +} diff --git a/algebird-test/src/test/scala/com/twitter/algebird/CheckProperties.scala b/algebird-test/src/test/scala-2/com/twitter/algebird/CheckProperties.scala similarity index 100% rename from algebird-test/src/test/scala/com/twitter/algebird/CheckProperties.scala rename to algebird-test/src/test/scala-2/com/twitter/algebird/CheckProperties.scala diff --git a/algebird-test/src/test/scala/com/twitter/algebird/CollectionSpecification.scala b/algebird-test/src/test/scala-2/com/twitter/algebird/CollectionSpecification.scala similarity index 94% rename from algebird-test/src/test/scala/com/twitter/algebird/CollectionSpecification.scala rename to algebird-test/src/test/scala-2/com/twitter/algebird/CollectionSpecification.scala index 2c641dbe6..b40c96285 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/CollectionSpecification.scala +++ b/algebird-test/src/test/scala-2/com/twitter/algebird/CollectionSpecification.scala @@ -35,7 +35,7 @@ class CollectionSpecification extends CheckProperties { implicit val equiv: Equiv[Map[String, Option[Int]]] = Equiv.fromFunction { (a, b) => val keys: Set[String] = a.keySet | b.keySet - keys.forall { key: String => + keys.forall { (key: String) => val v1: Int = a.getOrElse(key, None).getOrElse(0) val v2: Int = b.getOrElse(key, None).getOrElse(0) v1 == v2 @@ -95,12 +95,12 @@ class CollectionSpecification extends CheckProperties { implicit def scMapArb[K: Arbitrary, V: Arbitrary: Monoid]: Arbitrary[ScMap[K, V]] = Arbitrary { mapArb[K, V].arbitrary - .map { map: Map[K, V] => map: ScMap[K, V] } + .map((map: Map[K, V]) => map: ScMap[K, V]) } implicit def mMapArb[K: Arbitrary, V: Arbitrary: Monoid]: Arbitrary[MMap[K, V]] = Arbitrary { mapArb[K, V].arbitrary - .map { map: Map[K, V] => MMap(map.toSeq: _*): MMap[K, V] } + .map((map: Map[K, V]) => MMap(map.toSeq: _*): MMap[K, V]) } def mapPlusTimesKeys[M <: ScMap[Int, Int]](implicit @@ -336,26 +336,26 @@ class CollectionSpecification extends CheckProperties { ) property("AdaptiveVector[Int] has a semigroup") { - implicit val arb = Arbitrary(arbAV(2)) + implicit val arb: Arbitrary[AdaptiveVector[Int]] = Arbitrary(arbAV(2)) semigroupLaws[AdaptiveVector[Int]] } property("AdaptiveVector[Int] has a monoid") { // TODO: remove this equiv instance once #583 is resolved. - implicit val equiv = AdaptiveVector.denseEquiv[Int] - implicit val arb = Arbitrary(arbAV(0)) + implicit val equiv: Equiv[AdaptiveVector[Int]] = AdaptiveVector.denseEquiv[Int] + implicit val arb: Arbitrary[AdaptiveVector[Int]] = Arbitrary(arbAV(0)) monoidLaws[AdaptiveVector[Int]] } property("AdaptiveVector[Int] has a group") { - implicit val arb = Arbitrary(arbAV(1)) + implicit val arb: Arbitrary[AdaptiveVector[Int]] = Arbitrary(arbAV(1)) groupLaws[AdaptiveVector[Int]] } property("AdaptiveVector[String] has a monoid") { // TODO: remove this equiv instance once #583 is resolved. - implicit val equiv = AdaptiveVector.denseEquiv[String] - implicit val arb = Arbitrary(arbAV("")) + implicit val equiv: Equiv[AdaptiveVector[String]] = AdaptiveVector.denseEquiv[String] + implicit val arb: Arbitrary[AdaptiveVector[String]] = Arbitrary(arbAV("")) monoidLaws[AdaptiveVector[String]] } } diff --git a/algebird-test/src/test/scala-3/com/twitter/algebird/CheckProperties.scala b/algebird-test/src/test/scala-3/com/twitter/algebird/CheckProperties.scala new file mode 100644 index 000000000..ca215aba7 --- /dev/null +++ b/algebird-test/src/test/scala-3/com/twitter/algebird/CheckProperties.scala @@ -0,0 +1,10 @@ +package com.twitter.algebird + +import org.scalatestplus.scalacheck.Checkers +import org.scalatest.propspec.AnyPropSpec + +/** + * @author + * Mansur Ashraf. + */ +trait CheckProperties extends AnyPropSpec with Checkers diff --git a/algebird-test/src/test/scala/com/twitter/algebird/AbstractAlgebraTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/AbstractAlgebraTest.scala index c26d2dcc5..a9dc9fd03 100755 --- a/algebird-test/src/test/scala/com/twitter/algebird/AbstractAlgebraTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/AbstractAlgebraTest.scala @@ -9,18 +9,18 @@ class AbstractAlgebraTest extends CheckProperties with Matchers { property("A Monoid should be able to sum") { val monoid = implicitly[Monoid[Int]] - forAll { intList: List[Int] => intList.sum == monoid.sum(intList) } + forAll((intList: List[Int]) => intList.sum == monoid.sum(intList)) } property("A Ring should be able to product") { val ring = implicitly[Ring[Int]] - forAll { intList: List[Int] => intList.product == ring.product(intList) } + forAll((intList: List[Int]) => intList.product == ring.product(intList)) } property("An OptionMonoid should be able to sum") { val monoid = implicitly[Monoid[Option[Int]]] - forAll { intList: List[Option[Int]] => + forAll { (intList: List[Option[Int]]) => val flattenedList = intList.flatten val expectedResult = if (flattenedList.nonEmpty) Some(flattenedList.sum) else None @@ -31,7 +31,7 @@ class AbstractAlgebraTest extends CheckProperties with Matchers { property("An OptionMonoid based on a Semigroup should be able to sum") { val maxMonoid = implicitly[Monoid[Option[Max[Int]]]] val minMonoid = implicitly[Monoid[Option[Min[Int]]]] - forAll { intList: List[Option[Int]] => + forAll { (intList: List[Option[Int]]) => val minList = intList.map { case Some(x) => Some(Min(x)) case None => None @@ -55,7 +55,7 @@ class AbstractAlgebraTest extends CheckProperties with Matchers { property("IndexedSeq should sum") { forAll { (lIndexedSeq: IndexedSeq[Int]) => - val rIndexedSeq = lIndexedSeq.map(_ => scala.util.Random.nextInt) + val rIndexedSeq = lIndexedSeq.map(_ => scala.util.Random.nextInt()) (lIndexedSeq.size == rIndexedSeq.size) ==> { val leftBase = lIndexedSeq.map(Max(_)) val rightBase = rIndexedSeq.map(Max(_)) @@ -84,7 +84,7 @@ class AbstractAlgebraTest extends CheckProperties with Matchers { property("An ArrayMonoid should sum") { val monoid = new ArrayMonoid[Int] - forAll { intList: List[Int] => + forAll { (intList: List[Int]) => val (l, r) = intList.splitAt(intList.size / 2) val left = l.padTo(math.max(l.size, r.size), 0) val right = r.padTo(math.max(l.size, r.size), 0) @@ -97,7 +97,7 @@ class AbstractAlgebraTest extends CheckProperties with Matchers { property("An ArrayGroup should negate") { val arrayGroup = new ArrayGroup[Int] - forAll { intList: List[Int] => + forAll { (intList: List[Int]) => intList.map(-1 * _).toSeq == arrayGroup .negate(intList.toArray) .toSeq @@ -106,8 +106,12 @@ class AbstractAlgebraTest extends CheckProperties with Matchers { property("a user-defined product monoid should work") { case class Metrics(count: Int, largestValue: Option[Max[Int]]) - implicit val MetricsMonoid = Monoid(Metrics.apply _, Metrics.unapply _) - implicit val metricsGen = Arbitrary { + val unapply: Metrics => Option[(Int, Option[Max[Int]])] = { case Metrics(count, largestValue) => + Some((count, largestValue)) + } + implicit val MetricsMonoid: Monoid[Metrics] = + Monoid.apply[Metrics, Int, Option[Max[Int]]](Metrics.apply _, unapply) + implicit val metricsGen: Arbitrary[Metrics] = Arbitrary[Metrics] { for { count <- Gen.choose(0, 10000) largest <- Gen.oneOf[Option[Max[Int]]](None, Gen.choose(1, 100).map(n => Some(Max(n)))) diff --git a/algebird-test/src/test/scala/com/twitter/algebird/AggregatorLaws.scala b/algebird-test/src/test/scala/com/twitter/algebird/AggregatorLaws.scala index dd195d161..43dd5289d 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/AggregatorLaws.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/AggregatorLaws.scala @@ -59,22 +59,7 @@ class AggregatorTests extends AnyFunSuite { } } -class AggregatorLaws extends CheckProperties { - - implicit def aggregator[A, B, C](implicit - prepare: Arbitrary[A => B], - sg: Semigroup[B], - present: Arbitrary[B => C] - ): Arbitrary[Aggregator[A, B, C]] = Arbitrary { - for { - pp <- prepare.arbitrary - ps <- present.arbitrary - } yield new Aggregator[A, B, C] { - def prepare(a: A) = pp(a) - def semigroup = sg - def present(b: B) = ps(b) - } - } +class AggregatorLaws extends CheckProperties with AggregatorLawsCompat { property("composing before Aggregator is correct") { forAll { (in: List[Int], compose: (Int => Int), ag: Aggregator[Int, Int, Int]) => @@ -97,14 +82,6 @@ class AggregatorLaws extends CheckProperties { } } - property("Applicative composing two Aggregators is correct") { - forAll { (in: List[Int], ag1: Aggregator[Int, Set[Int], Int], ag2: Aggregator[Int, Unit, String]) => - type AggInt[T] = Aggregator[Int, _, T] - val c = Applicative.join[AggInt, Int, String](ag1, ag2) - in.isEmpty || c(in) == ((ag1(in), ag2(in))) - } - } - property("Aggregator.zip composing two Aggregators is correct") { forAll { ( @@ -126,7 +103,7 @@ class AggregatorLaws extends CheckProperties { } def checkNumericSum[T: Arbitrary](implicit num: Numeric[T]): Prop = - forAll { in: List[T] => + forAll { (in: List[T]) => val aggregator = Aggregator.numericSum[T] val ares = aggregator(in) val sres = in.map(num.toDouble).sum diff --git a/algebird-test/src/test/scala/com/twitter/algebird/ApplicativeProperties.scala b/algebird-test/src/test/scala/com/twitter/algebird/ApplicativeProperties.scala index ddcdf501c..cc2234bb8 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/ApplicativeProperties.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/ApplicativeProperties.scala @@ -54,7 +54,7 @@ class ApplicativeProperties extends CheckProperties { property("sequenceGen") { // This follows from the laws, so we are just testing // the implementation of sequenceGen against sequence here - forAll { ls: List[Option[Int]] => + forAll { (ls: List[Option[Int]]) => val res: Option[Vector[Int]] = Applicative.sequenceGen(ls) Applicative.sequence(ls).map(_.toVector) == res } @@ -62,27 +62,27 @@ class ApplicativeProperties extends CheckProperties { // Applicative algebras: import BaseProperties._ property("Applicative Semigroup") { - implicit val optSg = new ApplicativeSemigroup[Int, Option] - implicit val listSg = new ApplicativeSemigroup[String, List] + implicit val optSg: Semigroup[Option[Int]] = new ApplicativeSemigroup[Int, Option] + implicit val listSg: Semigroup[List[String]] = new ApplicativeSemigroup[String, List] // the + here is actually a cross-product, and testing sumOption blows up semigroupLaws[Option[Int]] && isAssociative[List[String]] } property("Applicative Monoid") { - implicit val optSg = new ApplicativeMonoid[Int, Option] - implicit val listSg = new ApplicativeMonoid[String, List] + implicit val optSg: Monoid[Option[Int]] = new ApplicativeMonoid[Int, Option] + implicit val listSg: Monoid[List[String]] = new ApplicativeMonoid[String, List] // the + here is actually a cross-product, and testing sumOption blows up monoidLaws[Option[Int]] && validZero[List[String]] } // These laws work for only "non-empty" monads property("Applicative Group") { - implicit val optSg = new ApplicativeGroup[Int, Some] + implicit val optSg: ApplicativeGroup[Int, Some] = new ApplicativeGroup[Int, Some] groupLaws[Some[Int]] } property("Applicative Ring") { - implicit val optSg = new ApplicativeRing[Int, Some] + implicit val optSg: ApplicativeRing[Int, Some] = new ApplicativeRing[Int, Some] ringLaws[Some[Int]] } } diff --git a/algebird-test/src/test/scala/com/twitter/algebird/AveragedValueLaws.scala b/algebird-test/src/test/scala/com/twitter/algebird/AveragedValueLaws.scala index 70107ed76..d9c0a8d21 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/AveragedValueLaws.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/AveragedValueLaws.scala @@ -26,7 +26,7 @@ class AveragedValueLaws extends CheckProperties { } property("AveragedValue.aggregator returns the average") { - forAll { v: NonEmptyVector[Double] => approxEq(1e-10)(avg(v.items), AveragedValue.aggregator(v.items)) } + forAll((v: NonEmptyVector[Double]) => approxEq(1e-10)(avg(v.items), AveragedValue.aggregator(v.items))) } property("AveragedValue instances subtract") { @@ -42,7 +42,7 @@ class AveragedValueLaws extends CheckProperties { } property("AveragedValue works by + or sumOption") { - forAll { v: NonEmptyVector[Double] => + forAll { (v: NonEmptyVector[Double]) => val avgs = v.items.map(AveragedValue(_)) val sumOpt = Semigroup.sumOption[AveragedValue](avgs).get val byPlus = avgs.reduce(_ + _) @@ -53,7 +53,7 @@ class AveragedValueLaws extends CheckProperties { } def numericAggregatorTest[T: Numeric: Arbitrary]: Prop = - forAll { v: NonEmptyVector[T] => + forAll { (v: NonEmptyVector[T]) => val averaged = AveragedValue.numericAggregator[T].apply(v.items) approxEq(1e-10)(avg(v.items), averaged) } diff --git a/algebird-test/src/test/scala/com/twitter/algebird/BloomFilterTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/BloomFilterTest.scala index 46ebc6826..9abc3230c 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/BloomFilterTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/BloomFilterTest.scala @@ -250,13 +250,13 @@ class BloomFilterProperties extends ApproximateProperties("BloomFilter") { for (falsePositiveRate <- List(0.1, 0.01, 0.001)) { property(s"has small false positive rate with false positive rate = $falsePositiveRate") = { - implicit val intGen = Gen.choose(1, 1000) + implicit val intGen: Gen[Int] = Gen.choose(1, 1000) toProp(new BloomFilterFalsePositives[Int](falsePositiveRate), 50, 50, 0.01) } } property("approximate cardinality") = { - implicit val intGen = Gen.choose(1, 1000) + implicit val intGen: Gen[Int] = Gen.choose(1, 1000) toProp(new BloomFilterCardinality[Int], 50, 1, 0.01) } } @@ -269,14 +269,14 @@ class BloomFilterTest extends AnyWordSpec with Matchers { "be possible to create from an iterator" in { val bfMonoid = new BloomFilterMonoid[String](RAND.nextInt(5) + 1, RAND.nextInt(64) + 32) - val entries = (0 until 100).map(_ => RAND.nextInt.toString) + val entries = (0 until 100).map(_ => RAND.nextInt().toString) val bf = bfMonoid.create(entries.iterator) assert(bf.isInstanceOf[BF[String]]) } "be possible to create from a sequence" in { val bfMonoid = new BloomFilterMonoid[String](RAND.nextInt(5) + 1, RAND.nextInt(64) + 32) - val entries = (0 until 100).map(_ => RAND.nextInt.toString) + val entries = (0 until 100).map(_ => RAND.nextInt().toString) val bf = bfMonoid.create(entries: _*) assert(bf.isInstanceOf[BF[String]]) } @@ -285,7 +285,7 @@ class BloomFilterTest extends AnyWordSpec with Matchers { (0 to 100).foreach { _ => val bfMonoid = new BloomFilterMonoid[String](RAND.nextInt(5) + 1, RAND.nextInt(64) + 32) val numEntries = 5 - val entries = (0 until numEntries).map(_ => RAND.nextInt.toString) + val entries = (0 until numEntries).map(_ => RAND.nextInt().toString) val bf = bfMonoid.create(entries: _*) entries.foreach(i => assert(bf.contains(i.toString).isTrue)) @@ -335,7 +335,7 @@ class BloomFilterTest extends AnyWordSpec with Matchers { (0 to 10).foreach { _ => val aggregator = BloomFilterAggregator[String](RAND.nextInt(5) + 1, RAND.nextInt(64) + 32) val numEntries = 5 - val entries = (0 until numEntries).map(_ => RAND.nextInt.toString) + val entries = (0 until numEntries).map(_ => RAND.nextInt().toString) val bf = aggregator(entries) entries.foreach(i => assert(bf.contains(i.toString).isTrue)) @@ -382,7 +382,7 @@ class BloomFilterTest extends AnyWordSpec with Matchers { (0 to 100).foreach { _ => val bfMonoid = new BloomFilterMonoid[String](RAND.nextInt(5) + 1, RAND.nextInt(64) + 32) val numEntries = 5 - val entries = (0 until numEntries).map(_ => RAND.nextInt.toString) + val entries = (0 until numEntries).map(_ => RAND.nextInt().toString) val bf = bfMonoid.create(entries: _*) entries .map(entry => (entry, bfMonoid.create(entry))) diff --git a/algebird-test/src/test/scala/com/twitter/algebird/BytesSpec.scala b/algebird-test/src/test/scala/com/twitter/algebird/BytesSpec.scala index 989811262..611798fce 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/BytesSpec.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/BytesSpec.scala @@ -79,7 +79,7 @@ class BytesSpec extends AnyWordSpec with Matchers with ScalaCheckDrivenPropertyC // Beyond the standard contract of `hashCode` (which is covered above), we also want small changes to an instance // to result in a different hash code. The verifications below (combined with the generator-driven test setup) // are a naive smoke test. - val appended = Bytes(bytes1.array :+ random.nextInt.toByte) + val appended = Bytes(bytes1.array :+ random.nextInt().toByte) bytes1.hashCode shouldNot be(appended.hashCode) if (bytes1.array.length > 1) { diff --git a/algebird-test/src/test/scala/com/twitter/algebird/CorrelationLaws.scala b/algebird-test/src/test/scala/com/twitter/algebird/CorrelationLaws.scala index ea55c60b9..76e711738 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/CorrelationLaws.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/CorrelationLaws.scala @@ -73,7 +73,7 @@ class CorrelationLaws extends CheckProperties { } property("the swap method on moments works as you'd think") { - forAll { l: List[(Double, Double)] => + forAll { (l: List[(Double, Double)]) => val swapped = CorrelationAggregator(l).swap val fn: ((Double, Double)) => (Double, Double) = { tup => tup.swap } diff --git a/algebird-test/src/test/scala/com/twitter/algebird/CountMinSketchTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/CountMinSketchTest.scala index f302dbf2d..5b35d30fc 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/CountMinSketchTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/CountMinSketchTest.scala @@ -38,44 +38,44 @@ class CmsLaws extends CheckProperties { } property("CountMinSketch[Short] is a Monoid") { - implicit val cmsMonoid = monoid[Short] - implicit val cmsGen = createArbitrary[Short](cmsMonoid) + implicit val cmsMonoid: CMSMonoid[Short] = monoid[Short] + implicit val cmsGen: Arbitrary[CMS[Short]] = createArbitrary[Short](cmsMonoid) commutativeMonoidLaws[CMS[Short]] } property("CountMinSketch[Int] is a Monoid") { - implicit val cmsMonoid = monoid[Int] - implicit val cmsGen = createArbitrary[Int](cmsMonoid) + implicit val cmsMonoid: CMSMonoid[Int] = monoid[Int] + implicit val cmsGen: Arbitrary[CMS[Int]] = createArbitrary[Int](cmsMonoid) commutativeMonoidLaws[CMS[Int]] } property("CountMinSketch[Long] is a Monoid") { - implicit val cmsMonoid = monoid[Long] - implicit val cmsGen = createArbitrary[Long](cmsMonoid) + implicit val cmsMonoid: CMSMonoid[Long] = monoid[Long] + implicit val cmsGen: Arbitrary[CMS[Long]] = createArbitrary[Long](cmsMonoid) commutativeMonoidLaws[CMS[Long]] } property("CountMinSketch[BigInt] is a Monoid") { - implicit val cmsMonoid = monoid[BigInt] - implicit val cmsGen = createArbitrary[BigInt](cmsMonoid) + implicit val cmsMonoid: CMSMonoid[BigInt] = monoid[BigInt] + implicit val cmsGen: Arbitrary[CMS[BigInt]] = createArbitrary[BigInt](cmsMonoid) commutativeMonoidLaws[CMS[BigInt]] } property("CountMinSketch[BigDecimal] is a Monoid") { - implicit val cmsMonoid = monoid[BigDecimal] - implicit val cmsGen = createArbitrary[BigDecimal](cmsMonoid) + implicit val cmsMonoid: CMSMonoid[BigDecimal] = monoid[BigDecimal] + implicit val cmsGen: Arbitrary[CMS[BigDecimal]] = createArbitrary[BigDecimal](cmsMonoid) commutativeMonoidLaws[CMS[BigDecimal]] } property("CountMinSketch[String] is a Monoid") { - implicit val cmsMonoid = monoid[String] - implicit val cmsGen = cmsArb(cmsMonoid)(_.toString) + implicit val cmsMonoid: CMSMonoid[String] = monoid[String] + implicit val cmsGen: Arbitrary[CMS[String]] = cmsArb(cmsMonoid)(_.toString) commutativeMonoidLaws[CMS[String]] } property("CountMinSketch[Bytes] is a commutative monoid") { - implicit val cmsMonoid = monoid[Bytes] - implicit val cmsGen = cmsArb(cmsMonoid)(CmsLaws.int2Bytes(_)) + implicit val cmsMonoid: CMSMonoid[Bytes] = monoid[Bytes] + implicit val cmsGen: Arbitrary[CMS[Bytes]] = cmsArb(cmsMonoid)(CmsLaws.int2Bytes(_)) commutativeMonoidLaws[CMS[Bytes]] } } @@ -104,44 +104,44 @@ class TopPctCmsLaws extends CheckProperties { } property("TopPctCms[Short] is a Monoid") { - implicit val cmsMonoid = monoid[Short] - implicit val cmsGen = createArbitrary[Short](cmsMonoid) + implicit val cmsMonoid: TopCMSMonoid[Short] = monoid[Short] + implicit val cmsGen: Arbitrary[TopCMS[Short]] = createArbitrary[Short](cmsMonoid) commutativeMonoidLaws[TopCMS[Short]] } property("TopPctCms[Int] is a Monoid") { - implicit val cmsMonoid = monoid[Int] - implicit val cmsGen = createArbitrary[Int](cmsMonoid) + implicit val cmsMonoid: TopCMSMonoid[Int] = monoid[Int] + implicit val cmsGen: Arbitrary[TopCMS[Int]] = createArbitrary[Int](cmsMonoid) commutativeMonoidLaws[TopCMS[Int]] } property("TopPctCms[Long] is a Monoid") { - implicit val cmsMonoid = monoid[Long] - implicit val cmsGen = createArbitrary[Long](cmsMonoid) + implicit val cmsMonoid: TopCMSMonoid[Long] = monoid[Long] + implicit val cmsGen: Arbitrary[TopCMS[Long]] = createArbitrary[Long](cmsMonoid) commutativeMonoidLaws[TopCMS[Long]] } property("TopPctCms[BigInt] is a Monoid") { - implicit val cmsMonoid = monoid[BigInt] - implicit val cmsGen = createArbitrary[BigInt](cmsMonoid) + implicit val cmsMonoid: TopCMSMonoid[BigInt] = monoid[BigInt] + implicit val cmsGen: Arbitrary[TopCMS[BigInt]] = createArbitrary[BigInt](cmsMonoid) commutativeMonoidLaws[TopCMS[BigInt]] } property("TopPctCms[BigDecimal] is a Monoid") { - implicit val cmsMonoid = monoid[BigDecimal] - implicit val cmsGen = createArbitrary[BigDecimal](cmsMonoid) + implicit val cmsMonoid: TopCMSMonoid[BigDecimal] = monoid[BigDecimal] + implicit val cmsGen: Arbitrary[TopCMS[BigDecimal]] = createArbitrary[BigDecimal](cmsMonoid) commutativeMonoidLaws[TopCMS[BigDecimal]] } property("TopPctCms[String] is a Monoid") { - implicit val cmsMonoid = monoid[String] - implicit val cmsGen = topCmsArb(cmsMonoid)(_.toString) + implicit val cmsMonoid: TopCMSMonoid[String] = monoid[String] + implicit val cmsGen: Arbitrary[TopCMS[String]] = topCmsArb(cmsMonoid)(_.toString) commutativeMonoidLaws[TopCMS[String]] } property("TopPctCms[Bytes] is a Monoid") { - implicit val cmsMonoid = monoid[Bytes] - implicit val cmsGen = topCmsArb(cmsMonoid)(CmsLaws.int2Bytes(_)) + implicit val cmsMonoid: TopCMSMonoid[Bytes] = monoid[Bytes] + implicit val cmsGen: Arbitrary[TopCMS[Bytes]] = topCmsArb(cmsMonoid)(CmsLaws.int2Bytes(_)) commutativeMonoidLaws[TopCMS[Bytes]] } } @@ -182,32 +182,11 @@ class CMSInstanceTest extends AnyWordSpec with Matchers with ScalaCheckDrivenPro * Verifies contramap functionality, which allows us to translate `CMSHasher[K]` into `CMSHasher[L]`, given * `f: L => K`. */ -class CMSContraMapSpec extends AnyWordSpec with Matchers with ScalaCheckDrivenPropertyChecks { - - "translates CMSHasher[K] into CMSHasher[L], given a function f: L => K" in { - // Given a "source" CMSHasher[K] - val sourceHasher: CMSHasher[String] = CMSHasher.CMSHasherString - // and a translation function from an unsupported type L (here: Seq[Byte]) to K - def f(bytes: Seq[Byte]): String = new String(bytes.toArray[Byte], "UTF-8") - - // When we run contramap on a CMSHasher[K] supplying f, - // then the result should be a CMSHasher[L]... - val targetHasher: CMSHasher[Seq[Byte]] = - sourceHasher.contramap((d: Seq[Byte]) => f(d)) - targetHasher shouldBe an[CMSHasher[ - _ - ]] // Can't test CMSHasher[Seq[Byte]] specifically because of type erasure. - - // ...and hashing should work correctly (this is only a smoke test). - val a = 4 - val b = 0 - val width = 1234 - val x = Array(113.toByte).toSeq // same as Seq(133.toByte) - val result = targetHasher.hash(a, b, width)(x) - val expected = sourceHasher.hash(a, b, width)("q") - result should be(expected) - result should be(434) - } +class CMSContraMapSpec + extends AnyWordSpec + with Matchers + with ScalaCheckDrivenPropertyChecks + with CMSContraMapSpecCompat { "supports, via contramap, creating CMS monoids for such types K that are not supported out of the box" in { // Given a "source" CMSHasher[K] which is supported out of the box @@ -329,7 +308,7 @@ object CmsProperty { } } -abstract class CmsFrequencyProperty[K: CMSHasher: Gen] extends CmsProperty { +abstract class CmsFrequencyProperty[K: CMSHasher: Gen] extends CmsProperty[K] { type Exact = Vector[K] type Approx = CMS[K] @@ -565,7 +544,7 @@ abstract class CMSTest[K: CMSHasher](toK: Int => K) .filter(_._2 < minHhCount) .keys .toSet - infrequent.intersect(estimatedHhs) should be('empty) + infrequent.intersect(estimatedHhs) should be(Symbol("empty")) } "(when adding CMS instances) drop old heavy hitters when new heavy hitters replace them" in { diff --git a/algebird-test/src/test/scala/com/twitter/algebird/DecayedValueLaws.scala b/algebird-test/src/test/scala/com/twitter/algebird/DecayedValueLaws.scala index 404a2b410..2a45e66de 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/DecayedValueLaws.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/DecayedValueLaws.scala @@ -26,7 +26,7 @@ class DecayedValueLaws extends CheckProperties { forAll { (params: Params) => val rand = new scala.util.Random val data = (0 to params.count).map { t => - val noise = rand.nextDouble * params.maxNoise * rand.nextInt.signum + val noise = rand.nextDouble() * params.maxNoise * rand.nextInt().signum DecayedValue.build(params.mean + (params.mean * noise), t, params.halfLife) } val result = decayedMonoid.sum(data) diff --git a/algebird-test/src/test/scala/com/twitter/algebird/EventuallyTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/EventuallyTest.scala index e91482291..6c5552b54 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/EventuallyTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/EventuallyTest.scala @@ -15,7 +15,7 @@ class EventuallyRingLaws extends CheckProperties { Arbitrary(Gen.oneOf(lGen, rGen)) property("EventuallyRing is a Ring") { - implicit val eventuallyRing = + implicit val eventuallyRing: EventuallyRing[Long, Int] = new EventuallyRing[Long, Int](_.toLong)(_ > 10000) ringLaws[Either[Long, Int]] } @@ -32,7 +32,7 @@ class EventuallyRingLaws extends CheckProperties { case (Left(a), Right(b)) => a == (b.toLong) } Prop.forAll { (pred: Int => Boolean) => - implicit val evRing = new EventuallyRing[Long, Int](_.toLong)(pred) + implicit val evRing: EventuallyRing[Long, Int] = new EventuallyRing[Long, Int](_.toLong)(pred) // TODO: convert to ringLaws https://github.com/twitter/algebird/issues/598 monoidLaws[Either[Long, Int]] } @@ -193,7 +193,7 @@ class EventuallyAggregatorLaws extends AnyPropSpec with ScalaCheckPropertyChecks * For HLL/Set, which is the common example, this is lawful. */ forAll { (in: List[Int], thresh: Int, rightAg: Aggregator[Int, List[Int], Int]) => - val pred = { x: List[Int] => x.lengthCompare(thresh) > 0 } + val pred = { (x: List[Int]) => x.lengthCompare(thresh) > 0 } val eventuallyAg = eventuallyAggregator(rightAg)(pred) eventuallyAg.semigroup diff --git a/algebird-test/src/test/scala/com/twitter/algebird/ExpHistLaws.scala b/algebird-test/src/test/scala/com/twitter/algebird/ExpHistLaws.scala index ae8023b2f..cfcdd2683 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/ExpHistLaws.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/ExpHistLaws.scala @@ -101,7 +101,7 @@ class ExpHistLaws extends AnyPropSpec with ScalaCheckPropertyChecks { def isPowerOfTwo(i: Long): Boolean = (i & -i) == i property("verify isPowerOfTwo") { - forAll { i: PosNum[Int] => + forAll { (i: PosNum[Int]) => val power = math.pow(2, i.value % 32).toLong assert(isPowerOfTwo(power)) } @@ -109,7 +109,7 @@ class ExpHistLaws extends AnyPropSpec with ScalaCheckPropertyChecks { // The next two properties are invariants from the paper. property("Invariant 1: relative error bound applies as old buckets expire") { - forAll { hist: ExpHist => + forAll { (hist: ExpHist) => val numBuckets = hist.buckets.size // sequence of histograms, each with one more oldest bucket @@ -124,7 +124,7 @@ class ExpHistLaws extends AnyPropSpec with ScalaCheckPropertyChecks { } property("Invariant 2: bucket sizes are nondecreasing powers of two") { - forAll { e: ExpHist => + forAll { (e: ExpHist) => assert(e.buckets.forall(b => isPowerOfTwo(b.size))) // sizes are nondecreasing: @@ -134,11 +134,11 @@ class ExpHistLaws extends AnyPropSpec with ScalaCheckPropertyChecks { } property("Total tracked by e is the sum of all bucket sizes") { - forAll { e: ExpHist => assert(e.buckets.map(_.size).sum == e.total) } + forAll((e: ExpHist) => assert(e.buckets.map(_.size).sum == e.total)) } property("ExpHist bucket sizes are the l-canonical rep of the tracked total") { - forAll { e: ExpHist => assert(e.buckets.map(_.size) == Canonical.bucketsFromLong(e.total, e.conf.l)) } + forAll((e: ExpHist) => assert(e.buckets.map(_.size) == Canonical.bucketsFromLong(e.total, e.conf.l))) } property("adding i results in upperBoundSum == i") { diff --git a/algebird-test/src/test/scala/com/twitter/algebird/FirstLaws.scala b/algebird-test/src/test/scala/com/twitter/algebird/FirstLaws.scala index 03a856206..de9880db7 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/FirstLaws.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/FirstLaws.scala @@ -7,7 +7,7 @@ import org.scalacheck.Prop.forAll class FirstLaws extends CheckProperties { property("First should sum properly") { - forAll { v: NonEmptyVector[First[Int]] => + forAll { (v: NonEmptyVector[First[Int]]) => val first = Semigroup.sumOption[First[Int]](v.items).get first == v.items.head } @@ -18,7 +18,7 @@ class FirstLaws extends CheckProperties { } property("First.aggregator returns the first item") { - forAll { v: NonEmptyVector[Int] => v.items.head == First.aggregator(v.items) } + forAll((v: NonEmptyVector[Int]) => v.items.head == First.aggregator(v.items)) } property("First[Int] is a semigroup")(semigroupLaws[First[Int]]) diff --git a/algebird-test/src/test/scala/com/twitter/algebird/GeneratedProductAlgebraLaws.scala b/algebird-test/src/test/scala/com/twitter/algebird/GeneratedProductAlgebraLaws.scala index 3b0c88894..8e3a42274 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/GeneratedProductAlgebraLaws.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/GeneratedProductAlgebraLaws.scala @@ -5,94 +5,96 @@ import com.twitter.algebird.BaseProperties._ class GeneratedProductAlgebraLaws extends CheckProperties { property("Product2Ring is a ring") { type T = (Int, Int) - implicit val ring = Ring[T, Int, Int](Tuple2.apply, Tuple2.unapply) + implicit val ring: Ring[T] = Ring[T, Int, Int](Tuple2.apply, Tuple2.unapply) ringLaws[T] && isCommutative[T] } property("Product3Ring is a ring") { type T = (Int, Int, Int) - implicit val ring = Ring[T, Int, Int, Int](Tuple3.apply, Tuple3.unapply) + implicit val ring: Ring[T] = Ring[T, Int, Int, Int](Tuple3.apply, Tuple3.unapply) ringLaws[T] && isCommutative[T] } property("Product4Ring is a ring") { type T = (Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int](Tuple4.apply, Tuple4.unapply) ringLaws[T] && isCommutative[T] } property("Product5Ring is a ring") { type T = (Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int](Tuple5.apply, Tuple5.unapply) ringLaws[T] && isCommutative[T] } property("Product6Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int](Tuple6.apply, Tuple6.unapply) ringLaws[T] && isCommutative[T] } property("Product7Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int, Int](Tuple7.apply, Tuple7.unapply) ringLaws[T] && isCommutative[T] } property("Product8Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int](Tuple8.apply, Tuple8.unapply) ringLaws[T] && isCommutative[T] } property("Product9Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int](Tuple9.apply, Tuple9.unapply) ringLaws[T] && isCommutative[T] } property("Product10Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int](Tuple10.apply, Tuple10.unapply) ringLaws[T] && isCommutative[T] } property("Product11Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int](Tuple11.apply, Tuple11.unapply) ringLaws[T] && isCommutative[T] } property("Product12Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int](Tuple12.apply, Tuple12.unapply) ringLaws[T] && isCommutative[T] } property("Product13Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int](Tuple13.apply, Tuple13.unapply) ringLaws[T] && isCommutative[T] } property("Product14Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int]( - Tuple14.apply, - Tuple14.unapply - ) + implicit val ring: Ring[T] = + Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int]( + Tuple14.apply, + Tuple14.unapply + ) ringLaws[T] && isCommutative[T] } property("Product15Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int]( - Tuple15.apply, - Tuple15.unapply - ) + implicit val ring: Ring[T] = + Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int]( + Tuple15.apply, + Tuple15.unapply + ) ringLaws[T] && isCommutative[T] } property("Product16Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int]( Tuple16.apply, Tuple16.unapply @@ -101,7 +103,7 @@ class GeneratedProductAlgebraLaws extends CheckProperties { } property("Product17Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int]( Tuple17.apply, Tuple17.unapply @@ -110,7 +112,7 @@ class GeneratedProductAlgebraLaws extends CheckProperties { } property("Product18Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int]( Tuple18.apply, Tuple18.unapply @@ -119,7 +121,7 @@ class GeneratedProductAlgebraLaws extends CheckProperties { } property("Product19Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = + implicit val ring: Ring[T] = Ring[T, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int]( Tuple19.apply, Tuple19.unapply @@ -129,7 +131,7 @@ class GeneratedProductAlgebraLaws extends CheckProperties { property("Product20Ring is a ring") { type T = (Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int, Int) - implicit val ring = Ring[ + implicit val ring: Ring[T] = Ring[ T, Int, Int, @@ -178,7 +180,7 @@ class GeneratedProductAlgebraLaws extends CheckProperties { Int, Int ) - implicit val ring = Ring[ + implicit val ring: Ring[T] = Ring[ T, Int, Int, @@ -229,7 +231,7 @@ class GeneratedProductAlgebraLaws extends CheckProperties { Int, Int ) - implicit val ring = Ring[ + implicit val ring: Ring[T] = Ring[ T, Int, Int, diff --git a/algebird-test/src/test/scala/com/twitter/algebird/HyperLogLogTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/HyperLogLogTest.scala index 8d3904751..c25225dab 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/HyperLogLogTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/HyperLogLogTest.scala @@ -73,7 +73,7 @@ class HyperLogLogLaws extends CheckProperties { * We can't change the way Array[Byte] was hashed without breaking serialized HLLs */ property("HyperLogLog.hash matches reference") { - Prop.forAll { a: Array[Byte] => HyperLogLog.hash(a).toSeq == ReferenceHyperLogLog.hash(a).toSeq } + Prop.forAll((a: Array[Byte]) => HyperLogLog.hash(a).toSeq == ReferenceHyperLogLog.hash(a).toSeq) } property("HyperLogLog.j and rhow match reference") { @@ -200,13 +200,13 @@ abstract class LargeSetSizeAggregatorProperty[T: Gen](bits: Int) extends SetSize } } -class SmallBytesSetSizeAggregatorProperty[T <% Array[Byte]: Gen](bits: Int) +class SmallBytesSetSizeAggregatorProperty[T: Gen](bits: Int)(implicit ev: T => Array[Byte]) extends SmallSetSizeAggregatorProperty[T] { def makeApproximate(s: Set[T]): Long = SetSizeAggregator[T](bits, maxSetSize).apply(s) } -class LargeBytesSetSizeAggregatorProperty[T <% Array[Byte]: Gen](bits: Int) +class LargeBytesSetSizeAggregatorProperty[T: Gen](bits: Int)(implicit ev: T => Array[Byte]) extends LargeSetSizeAggregatorProperty[T](bits) { def makeApproximate(s: Set[T]): Long = SetSizeAggregator[T](bits, maxSetSize).apply(s) @@ -286,9 +286,9 @@ class HyperLogLogTest extends AnyWordSpec with Matchers { val r: ju.Random = new java.util.Random def exactCount[T](it: Iterable[T]): Int = it.toSet.size - def approxCount[T <% Array[Byte]](bits: Int, it: Iterable[T]): Double = { + def approxCount[T](bits: Int, it: Iterable[T])(implicit ev: T => Array[Byte]): Double = { val hll = new HyperLogLogMonoid(bits) - hll.sizeOf(hll.sum(it.map(hll.create(_)))).estimate.toDouble + hll.sizeOf(hll.sum(it.map(elm => hll.create(ev(elm))))).estimate.toDouble } def aveErrorOf(bits: Int): Double = 1.04 / scala.math.sqrt(1 << bits) diff --git a/algebird-test/src/test/scala/com/twitter/algebird/IntervalLaws.scala b/algebird-test/src/test/scala/com/twitter/algebird/IntervalLaws.scala index 9c8928bc3..4b2c53d29 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/IntervalLaws.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/IntervalLaws.scala @@ -19,7 +19,7 @@ package com.twitter.algebird import org.scalacheck.Prop._ import com.twitter.algebird.scalacheck.PosNum -class IntervalLaws extends CheckProperties { +class IntervalLaws extends CheckProperties with IntervalLawsCompat { import com.twitter.algebird.scalacheck.arbitrary._ import com.twitter.algebird.Interval.GenIntersection @@ -36,14 +36,14 @@ class IntervalLaws extends CheckProperties { } property("[x, x + 1) contains x") { - forAll { y: Int => + forAll { (y: Int) => val x = y.asInstanceOf[Long] Interval.leftClosedRightOpen(x, x + 1).contains(x) } } property("[x, x + 1] contains x, x+1") { - forAll { y: Int => + forAll { (y: Int) => val x = y.asInstanceOf[Long] val intr = Interval.closed(x, x + 1) intr.contains(x) && @@ -53,7 +53,7 @@ class IntervalLaws extends CheckProperties { } } property("(x, x + 2) contains x+1") { - forAll { y: Int => + forAll { (y: Int) => val x = y.asInstanceOf[Long] val intr = Interval.open(x, x + 2) intr.contains(x + 1) && @@ -63,26 +63,26 @@ class IntervalLaws extends CheckProperties { } property("(x, x + 1] contains x + 1") { - forAll { y: Int => + forAll { (y: Int) => val x = y.asInstanceOf[Long] Interval.leftOpenRightClosed(x, x + 1).contains(x + 1) } } property("[x, x + 1) does not contain x + 1") { - forAll { x: Int => !Interval.leftClosedRightOpen(x, x + 1).contains(x + 1) } + forAll((x: Int) => !Interval.leftClosedRightOpen(x, x + 1).contains(x + 1)) } property("(x, x + 1] does not contain x") { - forAll { x: Int => !Interval.leftOpenRightClosed(x, x + 1).contains(x) } + forAll((x: Int) => !Interval.leftOpenRightClosed(x, x + 1).contains(x)) } property("[x, x) is empty") { - forAll { x: Int => Interval.leftClosedRightOpen(x, x).isEmpty } + forAll((x: Int) => Interval.leftClosedRightOpen(x, x).isEmpty) } property("(x, x] is empty") { - forAll { x: Int => Interval.leftOpenRightClosed(x, x).isEmpty } + forAll((x: Int) => Interval.leftOpenRightClosed(x, x).isEmpty) } property("[x, y).isEmpty == (x >= y)") { @@ -298,32 +298,6 @@ class IntervalLaws extends CheckProperties { } } } - property("if boundedLeast is none, we are Universe, Upper or isEmpty is true") { - forAll { (a: Interval[Long]) => - a.boundedLeast match { - case Some(_) => true - case None => - a.isEmpty || (a match { - case _: Upper[_] => true - case Universe() => true - case _ => false - }) - } - } - } - property("if boundedGreatest is none, we are Universe, Lower or isEmpty is true") { - forAll { (a: Interval[Long]) => - a.boundedGreatest match { - case Some(_) => true - case None => - a.isEmpty || (a match { - case _: Lower[_] => true - case Universe() => true - case _ => false - }) - } - } - } property("invalid closed bounds are empty") { forAll { (a: Long, b: Long) => diff --git a/algebird-test/src/test/scala/com/twitter/algebird/LastLaws.scala b/algebird-test/src/test/scala/com/twitter/algebird/LastLaws.scala index 49b54dce2..f2159d4db 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/LastLaws.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/LastLaws.scala @@ -7,7 +7,7 @@ import org.scalacheck.Prop.forAll class LastLaws extends CheckProperties { property("Last should sum properly") { - forAll { v: NonEmptyVector[Last[Int]] => + forAll { (v: NonEmptyVector[Last[Int]]) => val last = Semigroup.sumOption[Last[Int]](v.items).get last == v.items.last } @@ -18,7 +18,7 @@ class LastLaws extends CheckProperties { } property("Last.aggregator returns the last item") { - forAll { v: NonEmptyVector[Int] => v.items.last == Last.aggregator(v.items) } + forAll((v: NonEmptyVector[Int]) => v.items.last == Last.aggregator(v.items)) } property("Last[Int] is a Semigroup")(semigroupLaws[Last[Int]]) diff --git a/algebird-test/src/test/scala/com/twitter/algebird/MaxLaws.scala b/algebird-test/src/test/scala/com/twitter/algebird/MaxLaws.scala index 361ed2a45..a38b7573f 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/MaxLaws.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/MaxLaws.scala @@ -15,7 +15,7 @@ class MaxLaws extends CheckProperties { } def maxSemiGroupTest[T: Arbitrary: Ordering]: Prop = - forAll { v: NonEmptyVector[T] => + forAll { (v: NonEmptyVector[T]) => val maxItems = v.items.map(Max(_)) v.items.max == Max.semigroup[T].combineAllOption(maxItems).get.get } @@ -31,7 +31,7 @@ class MaxLaws extends CheckProperties { property("Max.{ +, max } works on ints")(maxTest[Int]) property("Max.aggregator returns the maximum item") { - forAll { v: NonEmptyVector[Int] => v.items.max == Max.aggregator[Int].apply(v.items) } + forAll((v: NonEmptyVector[Int]) => v.items.max == Max.aggregator[Int].apply(v.items)) } property("Max.semigroup[Int] returns the maximum item") { diff --git a/algebird-test/src/test/scala/com/twitter/algebird/MinHasherTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/MinHasherTest.scala index 1ec842405..341cc2fd5 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/MinHasherTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/MinHasherTest.scala @@ -39,16 +39,16 @@ class MinHasherSpec extends AnyWordSpec with Matchers { val sharedFraction = 1 - uniqueFraction val unique1 = 1 .to((s * uniqueFraction).toInt) - .map(_ => math.random) + .map(_ => Math.random()) .toSet val unique2 = 1 .to((s * uniqueFraction).toInt) - .map(_ => math.random) + .map(_ => Math.random()) .toSet val shared = 1 .to((s * sharedFraction).toInt) - .map(_ => math.random) + .map(_ => Math.random()) .toSet (unique1 ++ shared, unique2 ++ shared) } diff --git a/algebird-test/src/test/scala/com/twitter/algebird/MinLaws.scala b/algebird-test/src/test/scala/com/twitter/algebird/MinLaws.scala index 5b5d379a7..eda8b1c37 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/MinLaws.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/MinLaws.scala @@ -15,7 +15,7 @@ class MinLaws extends CheckProperties { } def minSemigroupTest[T: Arbitrary: Ordering]: Prop = - forAll { v: NonEmptyVector[T] => + forAll { (v: NonEmptyVector[T]) => val minItems = v.items.map(Min(_)) v.items.min == Min.semigroup[T].combineAllOption(minItems).get.get } @@ -30,7 +30,7 @@ class MinLaws extends CheckProperties { } property("Min.aggregator returns the minimum item") { - forAll { v: NonEmptyVector[Int] => v.items.min == Min.aggregator[Int].apply(v.items) } + forAll((v: NonEmptyVector[Int]) => v.items.min == Min.aggregator[Int].apply(v.items)) } property("Min.semigroup[Int] returns the minimum item") { diff --git a/algebird-test/src/test/scala/com/twitter/algebird/PredecessibleProperties.scala b/algebird-test/src/test/scala/com/twitter/algebird/PredecessibleProperties.scala index 8023122a2..8870a8d0d 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/PredecessibleProperties.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/PredecessibleProperties.scala @@ -22,7 +22,7 @@ class PredecessibleProperties extends CheckProperties { property("Long is Predecessible")(laws[Long]) property("BigInt is Predecessible")(laws[BigInt]) property("Predecessible.fromPrevOrd[Int] is Predecessible") { - implicit val pred = + implicit val pred: Predecessible[Int] = Predecessible.fromPrevOrd[Int](IntegralPredecessible.prev(_)) laws[Int] } diff --git a/algebird-test/src/test/scala/com/twitter/algebird/QTreeTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/QTreeTest.scala index 90ce4deda..5ab7716a3 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/QTreeTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/QTreeTest.scala @@ -36,7 +36,7 @@ class QTreeLaws extends CheckProperties { class QTreeTest extends AnyWordSpec with Matchers { def randomList(n: Long): scala.collection.immutable.IndexedSeq[Double] = - (1L to n).map(_ => math.random) + (1L to n).map(_ => Math.random()) def buildQTree(k: Int, list: Seq[Double]): QTree[Double] = { val qtSemigroup = new QTreeSemigroup[Double](k) @@ -56,10 +56,10 @@ class QTreeTest extends AnyWordSpec with Matchers { s"QTree with elements (1 to $k)" should { val trueMedian = (1 + k) / 2 s"have median $trueMedian" in { - implicit val sg = new QTreeSemigroup[Unit](6) + implicit val sg: QTreeSemigroup[Unit] = new QTreeSemigroup[Unit](6) val list = (1 to k).map(_.toDouble) - val qtree = sg.sumOption(list.map(QTree.value(_))).get + val qtree: QTree[Unit] = sg.sumOption(list.map(QTree.value(_))).get val (lower, upper) = qtree.quantileBounds(0.5) assert(lower <= trueMedian && trueMedian <= upper) @@ -72,7 +72,7 @@ class QTreeTest extends AnyWordSpec with Matchers { "always contain the true quantile within its bounds" in { val list = randomList(10000) val qt = buildQTree(k, list) - val quantile = math.random + val quantile = Math.random() val (lower, upper) = qt.quantileBounds(quantile) val truth = trueQuantile(list, quantile) assert(truth >= lower) @@ -89,8 +89,8 @@ class QTreeTest extends AnyWordSpec with Matchers { "always contain the true range sum within its bounds" in { val list = randomList(10000) val qt = buildQTree(k, list) - val from = math.random - val to = math.random + val from = Math.random() + val to = Math.random() val (lower, upper) = qt.rangeSumBounds(from, to) val truth = trueRangeSum(list, from, to) assert(truth >= lower) diff --git a/algebird-test/src/test/scala/com/twitter/algebird/ResetTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/ResetTest.scala index f1b19bc22..18ddcb089 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/ResetTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/ResetTest.scala @@ -24,7 +24,7 @@ class ResetTest extends CheckProperties { implicit def resetArb[T: Arbitrary]: Arbitrary[ResetState[T]] = Arbitrary { Arbitrary.arbitrary[T].map { t => - if (scala.math.random < 0.1) { + if (Math.random() < 0.1) { ResetValue(t) } else { SetValue(t) diff --git a/algebird-test/src/test/scala/com/twitter/algebird/SemigroupTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/SemigroupTest.scala index c49776e87..aa27f2b9d 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/SemigroupTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/SemigroupTest.scala @@ -5,7 +5,7 @@ import org.scalatest.funsuite.AnyFunSuite class SemigroupTest extends AnyFunSuite with ScalaCheckPropertyChecks { test("Semigroup.maybePlus works") { - forAll { s: String => + forAll { (s: String) => assert(Semigroup.maybePlus(None, s) == s) assert(Semigroup.maybePlus(s, None) == s) } diff --git a/algebird-test/src/test/scala/com/twitter/algebird/SpaceSaverTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/SpaceSaverTest.scala index 922ad8534..276c74d21 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/SpaceSaverTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/SpaceSaverTest.scala @@ -40,7 +40,7 @@ class SpaceSaverLaws extends CheckProperties { } property("SpaceSaver can serialize/deserialize itself") { - forAll { ss: SpaceSaver[String] => + forAll { (ss: SpaceSaver[String]) => val ssAsBytes = SpaceSaver.toBytes(ss, SpaceSaverTest.stringToArrayByte) SpaceSaver .fromBytes(ssAsBytes, SpaceSaverTest.arrayByteToString) @@ -49,7 +49,7 @@ class SpaceSaverLaws extends CheckProperties { } property("SpaceSaver.fromBytes yield a failure on bad Array[Byte]") { - forAll { a: Array[Byte] => + forAll { (a: Array[Byte]) => try { val fromBytes = SpaceSaver.fromBytes(a, SpaceSaverTest.arrayByteToString) diff --git a/algebird-test/src/test/scala/com/twitter/algebird/SuccessibleProperties.scala b/algebird-test/src/test/scala/com/twitter/algebird/SuccessibleProperties.scala index 901305cea..e749bdc0e 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/SuccessibleProperties.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/SuccessibleProperties.scala @@ -25,7 +25,7 @@ class SuccessibleProperties extends CheckProperties { property("Long is Successible")(laws[Long]) property("BigInt is Successible")(laws[BigInt]) property("Successible.fromNextOrd[Int] is Successible") { - implicit val succ = + implicit val succ: Successible[Int] = Successible.fromNextOrd[Int](IntegralSuccessible.next(_)) laws[Int] } diff --git a/algebird-test/src/test/scala/com/twitter/algebird/SummingQueueTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/SummingQueueTest.scala index 07eeff67b..4b111dc2a 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/SummingQueueTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/SummingQueueTest.scala @@ -40,7 +40,7 @@ class SummingCacheTest extends CheckProperties { def test[K, V: Monoid](c: Capacity, items: List[(K, V)]): Boolean = { val sc = newCache[K, V](c) val mitems = items.map(Map(_)) - implicit val mapEq = mapEquiv[K, V] + implicit val mapEq: Equiv[Map[K, V]] = mapEquiv[K, V] StatefulSummerLaws.sumIsPreserved(sc, mitems) && StatefulSummerLaws.isFlushedIsConsistent(sc, mitems) } @@ -76,7 +76,7 @@ class SummingWithHitsCacheTest extends SummingCacheTest { forAll { (c: Capacity, values: List[Int]) => // Only run this when we have at least 2 items and non-zero cap (values.size > 1 && c.cap > 1) ==> { - val key = RAND.nextInt + val key = RAND.nextInt() val items = values.map((key, _)) val keyHits = getHits(c, items) !keyHits.exists(_ != 1) @@ -85,7 +85,7 @@ class SummingWithHitsCacheTest extends SummingCacheTest { } property("hit rates will always be 0 when cap is 0") { - forAll { items: List[(Int, Int)] => + forAll { (items: List[(Int, Int)]) => // Only run this when we have at least 2 items (items.size > 1) ==> { val keyHits = getHits(Capacity(0), items) @@ -110,7 +110,7 @@ class SummingQueueTest extends CheckProperties { val zeroCapQueue: SummingQueue[Int] = SummingQueue[Int](0) // passes all through property("0 capacity always returns") { - forAll { i: Int => zeroCapQueue(i) == Some(i) } + forAll((i: Int) => zeroCapQueue(i) == Some(i)) } val sb: SummingQueue[Int] = SummingQueue[Int](3) // buffers three at a time diff --git a/algebird-test/src/test/scala/com/twitter/algebird/WindowLawsTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/WindowLawsTest.scala index cf989aa1b..850a1ab55 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/WindowLawsTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/WindowLawsTest.scala @@ -21,7 +21,7 @@ class WindowLaws extends CheckProperties { property("Window obeys monoid laws using a group")(monoidLaws[Window[Int]]) property("Window obeys monoid laws using a monoid") { - implicit val mon = Window.monoid[String](5) + implicit val mon: WindowMonoid[String] = Window.monoid[String](5) monoidLaws[Window[String]] } } diff --git a/algebird-test/src/test/scala/com/twitter/algebird/immutable/BloomFilterTest.scala b/algebird-test/src/test/scala/com/twitter/algebird/immutable/BloomFilterTest.scala index 56f314067..ff4c70920 100644 --- a/algebird-test/src/test/scala/com/twitter/algebird/immutable/BloomFilterTest.scala +++ b/algebird-test/src/test/scala/com/twitter/algebird/immutable/BloomFilterTest.scala @@ -252,13 +252,13 @@ class ImmutableBloomFilterProperties extends ApproximateProperties("BloomFilter" for (falsePositiveRate <- List(0.1, 0.01, 0.001)) { property(s"has small false positive rate with false positive rate = $falsePositiveRate") = { - implicit val intGen = Gen.choose(1, 1000) + implicit val intGen: Gen[Int] = Gen.choose(1, 1000) toProp(new BloomFilterFalsePositives[Int](falsePositiveRate), 50, 50, 0.01) } } property("approximate cardinality") = { - implicit val intGen = Gen.choose(1, 1000) + implicit val intGen: Gen[Int] = Gen.choose(1, 1000) toProp(new BloomFilterCardinality[Int], 50, 1, 0.01) } } @@ -271,21 +271,21 @@ class ImmutableBloomFilterTest extends AnyWordSpec with Matchers { "be possible to create from an iterator" in { val bloomFilter = BloomFilter[String](RAND.nextInt(5) + 1, RAND.nextInt(64) + 32) - val entries = (0 until 100).map(_ => RAND.nextInt.toString) + val entries = (0 until 100).map(_ => RAND.nextInt().toString) val bf = bloomFilter.create(entries.iterator) assert(bf.isInstanceOf[bloomFilter.Hash]) } "be possible to create from a sequence" in { val bloomFilter = BloomFilter[String](RAND.nextInt(5) + 1, RAND.nextInt(64) + 32) - val entries = (0 until 100).map(_ => RAND.nextInt.toString) + val entries = (0 until 100).map(_ => RAND.nextInt().toString) val bf = bloomFilter.create(entries: _*) assert(bf.isInstanceOf[bloomFilter.Hash]) } "be possible to create from a BitSet" in { val bloomFilter = BloomFilter[String](RAND.nextInt(5) + 1, RAND.nextInt(64) + 32) - val entries = (0 until 100).map(_ => RAND.nextInt.toString) + val entries = (0 until 100).map(_ => RAND.nextInt().toString) val bf = bloomFilter.create(entries: _*) val instance = bloomFilter.fromBitSet(bf.toBitSet) @@ -300,7 +300,7 @@ class ImmutableBloomFilterTest extends AnyWordSpec with Matchers { "fail to create from a larger BitSet" in { val bloomFilter = BloomFilter[String](6, 0.01) - val entries = (0 until 6).map(_ => RAND.nextInt.toString) + val entries = (0 until 6).map(_ => RAND.nextInt().toString) val bf = bloomFilter.create(entries: _*) val instance = BloomFilter[String](6, 0.1).fromBitSet(bf.toBitSet) @@ -311,7 +311,7 @@ class ImmutableBloomFilterTest extends AnyWordSpec with Matchers { (0 to 100).foreach { _ => val bloomFilter = BloomFilter[String](RAND.nextInt(5) + 1, RAND.nextInt(64) + 32) val numEntries = 5 - val entries = (0 until numEntries).map(_ => RAND.nextInt.toString) + val entries = (0 until numEntries).map(_ => RAND.nextInt().toString) val bf = bloomFilter.create(entries: _*) entries.foreach { i => @@ -366,7 +366,7 @@ class ImmutableBloomFilterTest extends AnyWordSpec with Matchers { import bloomFilter.aggregator val numEntries = 5 - val entries = (0 until numEntries).map(_ => RAND.nextInt.toString) + val entries = (0 until numEntries).map(_ => RAND.nextInt().toString) val bf = aggregator(entries) entries.foreach(i => assert(bf.contains(i.toString).isTrue)) @@ -414,7 +414,7 @@ class ImmutableBloomFilterTest extends AnyWordSpec with Matchers { import bloomFilter._ val numEntries = 5 - val entries = (0 until numEntries).map(_ => RAND.nextInt.toString) + val entries = (0 until numEntries).map(_ => RAND.nextInt().toString) val bf = bloomFilter.create(entries: _*) entries .map(entry => (entry, bloomFilter.create(entry))) diff --git a/build.sbt b/build.sbt index ce803439b..1678aa138 100644 --- a/build.sbt +++ b/build.sbt @@ -9,13 +9,39 @@ val kindProjectorVersion = "0.13.2" val paradiseVersion = "2.1.1" val quasiquotesVersion = "2.1.0" val scalaTestVersion = "3.2.15" -val scalaTestPlusVersion = "3.1.0.0-RC2" -val scalacheckVersion = "1.15.2" +val scalaTestPlusFor211Version = "3.1.0.0-RC2" +val scalaTestPlusVersion = "3.2.11.0" +val scalacheckVersion = "1.15.3" +val scalacheckFor211Version = "1.15.2" val scalaCollectionCompat = "2.9.0" val utilVersion = "21.2.0" val sparkVersion = "2.4.8" def scalaVersionSpecificFolders(srcBaseDir: java.io.File, scalaVersion: String) = + CrossVersion.partialVersion(scalaVersion) match { + case Some((2, y)) if y <= 11 => + new java.io.File(s"${srcBaseDir.getPath}-2.12-") :: Nil + case Some((2, y)) if y <= 12 => + new java.io.File(s"${srcBaseDir.getPath}-2.12-") :: new java.io.File( + s"${srcBaseDir.getPath}-2.12+" + ) :: Nil + case Some((2, y)) if y >= 13 => + new java.io.File(s"${srcBaseDir.getPath}-2.13+") :: new java.io.File( + s"${srcBaseDir.getPath}-2.12+" + ) :: Nil + case Some((3, _)) => + new java.io.File(s"${srcBaseDir.getPath}-2.13+") :: new java.io.File( + s"${srcBaseDir.getPath}-2.12+" + ) :: Nil + case _ => Nil + } +// Workaround: skip compiling java api for Scala 3 because +// 1. it seems Scala 3 generates Monoid$ in later stage than Scala 2.x, +// which prevents javaapi from compiling due to missing symbol. +// 2. we cannot use `compileOrder := CompileOrder.ScalaThenJava` +// as algebird-core must compile CassandraMurmurHash.java BEFORE Scala, +// but at the same time algebird-core must compile javaapi AFTER Scala. +def scalaVersionSpecificJavaFolders(srcBaseDir: java.io.File, scalaVersion: String) = CrossVersion.partialVersion(scalaVersion) match { case Some((2, y)) if y <= 12 => new java.io.File(s"${srcBaseDir.getPath}-2.12-") :: Nil @@ -23,6 +49,14 @@ def scalaVersionSpecificFolders(srcBaseDir: java.io.File, scalaVersion: String) new java.io.File(s"${srcBaseDir.getPath}-2.13+") :: Nil case _ => Nil } +def scalaVersionSpecificJavaFoldersForTest(srcBaseDir: java.io.File, scalaVersion: String) = + CrossVersion.partialVersion(scalaVersion) match { + case Some((2, y)) if y <= 12 => + new java.io.File(s"${srcBaseDir.getPath}-2") :: new java.io.File(s"${srcBaseDir.getPath}-2.12-") :: Nil + case Some((2, y)) if y >= 13 => + new java.io.File(s"${srcBaseDir.getPath}-2") :: new java.io.File(s"${srcBaseDir.getPath}-2.13+") :: Nil + case _ => Nil + } def scalaBinaryVersion(scalaVersion: String) = scalaVersion match { case version if version.startsWith("2.11") => "2.11" @@ -31,6 +65,8 @@ def scalaBinaryVersion(scalaVersion: String) = scalaVersion match { case version => sys.error(s"unsupported scala version $version") } +def isScala3(scalaVersion: String) = scalaVersion.startsWith("3.") + def isScala212x(scalaVersion: String) = scalaBinaryVersion(scalaVersion) == "2.12" def isScala213x(scalaVersion: String) = scalaBinaryVersion(scalaVersion) == "2.13" @@ -101,9 +137,13 @@ val sharedSettings = Seq( (Test / scalaSource).value, scalaVersion.value ), - Compile / unmanagedSourceDirectories ++= scalaVersionSpecificFolders( + Compile / unmanagedSourceDirectories ++= scalaVersionSpecificJavaFolders( (Compile / javaSource).value, scalaVersion.value + ), + Test / unmanagedSourceDirectories ++= scalaVersionSpecificJavaFoldersForTest( + (Test / scalaSource).value, + scalaVersion.value ) ) ++ mimaSettings @@ -170,6 +210,34 @@ lazy val mimaSettings = Def.settings( ProblemFilters.exclude[IncompatibleSignatureProblem]("com.twitter.algebird.MinHasher.*") ) ) +// NOTE: After dropping Scala 2.11, we can remove src/main/scala-2.11 and share sources between scala 2.12, 2.13 and 3.x. + +val compilerExtraSettings = + Seq( + Compile / scalacOptions ++= { + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((3, _)) => Seq("-Ykind-projector:underscores") + case Some((2, 12 | 13)) => Seq("-Xsource:3", "-P:kind-projector:underscore-placeholders") + case _ => Seq.empty + } + }, + libraryDependencies ++= { + if (isScala3(scalaVersion.value)) { + Seq.empty + } else if (isScala213x(scalaVersion.value)) { + Seq( + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + compilerPlugin("org.typelevel" % "kind-projector" % kindProjectorVersion).cross(CrossVersion.full) + ) + } else { + Seq( + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + compilerPlugin(("org.scalamacros" % "paradise" % paradiseVersion).cross(CrossVersion.full)), + compilerPlugin("org.typelevel" % "kind-projector" % kindProjectorVersion).cross(CrossVersion.full) + ) + } + } + ) /** * This returns the previous jar we released that is compatible with the current. @@ -204,54 +272,51 @@ def module(name: String) = { .settings(sharedSettings ++ Seq(Keys.name := id, mimaPreviousArtifacts := previousVersion(name).toSet)) } -lazy val algebirdCore = module("core").settings( - crossScalaVersions += "2.13.10", - initialCommands := """ +lazy val algebirdCore = module("core") + .settings( + crossScalaVersions := Seq("2.11.12", "2.12.17", "2.13.10", "3.3.0"), + initialCommands := """ import com.twitter.algebird._ """.stripMargin('|'), + libraryDependencies ++= + Seq( + "com.googlecode.javaewah" % "JavaEWAH" % javaEwahVersion, + ("org.typelevel" %% "algebra" % algebraVersion).cross(CrossVersion.for3Use2_13), + "org.scalatest" %% "scalatest" % scalaTestVersion % "test", + "org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompat + ), + Compile / sourceGenerators += Def.task { + GenTupleAggregators.gen((Compile / sourceManaged).value) + }.taskValue, + // Scala 2.12's doc task was failing. + Compile / doc / sources ~= (_.filterNot(_.absolutePath.contains("javaapi"))), + Test / testOptions := Seq(Tests.Argument(TestFrameworks.JUnit, "-a")) + ) + .settings(compilerExtraSettings) + +val algebirdTestDependenciesSettings = Seq( libraryDependencies ++= Seq( - "com.googlecode.javaewah" % "JavaEWAH" % javaEwahVersion, - "org.typelevel" %% "algebra" % algebraVersion, - "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "org.scalatest" %% "scalatest" % scalaTestVersion % "test", - "org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompat - ) ++ { - if (isScala213x(scalaVersion.value)) { - Seq() - } else { - Seq(compilerPlugin(("org.scalamacros" % "paradise" % paradiseVersion).cross(CrossVersion.full))) - } - }, - addCompilerPlugin(("org.typelevel" % "kind-projector" % kindProjectorVersion).cross(CrossVersion.full)), - Compile / sourceGenerators += Def.task { - GenTupleAggregators.gen((Compile / sourceManaged).value) - }.taskValue, - // Scala 2.12's doc task was failing. - Compile / doc / sources ~= (_.filterNot(_.absolutePath.contains("javaapi"))), - Test / testOptions := Seq(Tests.Argument(TestFrameworks.JUnit, "-a")) + "org.scalatest" %% "scalatest" % scalaTestVersion + ) ++ (if (scalaVersion.value.startsWith("2.11")) + Seq( + "org.scalacheck" %% "scalacheck" % scalacheckFor211Version, + "org.scalatestplus" %% "scalatestplus-scalacheck" % scalaTestPlusFor211Version % "test" + ) + else + Seq( + "org.scalacheck" %% "scalacheck" % scalacheckVersion, + "org.scalatestplus" %% "scalacheck-1-15" % scalaTestPlusVersion % "test" + )) ) lazy val algebirdTest = module("test") .settings( Test / testOptions ++= Seq(Tests.Argument(TestFrameworks.ScalaCheck, "-verbosity", "4")), - crossScalaVersions += "2.13.10", - libraryDependencies ++= - Seq( - "org.scalacheck" %% "scalacheck" % scalacheckVersion, - "org.scalatest" %% "scalatest" % scalaTestVersion, - "org.scalatestplus" %% "scalatestplus-scalacheck" % scalaTestPlusVersion % "test" - ) ++ { - if (isScala213x(scalaVersion.value)) { - Seq() - } else { - Seq(compilerPlugin(("org.scalamacros" % "paradise" % paradiseVersion).cross(CrossVersion.full))) - } - }, - addCompilerPlugin( - ("org.typelevel" % "kind-projector" % kindProjectorVersion).cross(CrossVersion.full) - ) + crossScalaVersions := Seq("2.11.12", "2.12.17", "2.13.10", "3.3.0") ) + .settings(algebirdTestDependenciesSettings) + .settings(compilerExtraSettings) .dependsOn(algebirdCore) lazy val algebirdBenchmark = module("benchmark") diff --git a/project/build.properties b/project/build.properties index 22af2628c..40b3b8e7b 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.7.1 +sbt.version=1.9.0