org.apache.spark.mllib.regression.LinearRegressionModel Scala Examples
The following examples show how to use org.apache.spark.mllib.regression.LinearRegressionModel.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
| Source File: PMMLModelExportFactory.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.dmg.pmml.RegressionNormalizationMethodType
import org.apache.spark.mllib.classification.LogisticRegressionModel
import org.apache.spark.mllib.classification.SVMModel
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.regression.LassoModel
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.RidgeRegressionModel
private[mllib] object PMMLModelExportFactory {
def createPMMLModelExport(model: Any): PMMLModelExport = {
model match {
case kmeans: KMeansModel =>
new KMeansPMMLModelExport(kmeans)
case linear: LinearRegressionModel =>
new GeneralizedLinearPMMLModelExport(linear, "linear regression")
case ridge: RidgeRegressionModel =>
new GeneralizedLinearPMMLModelExport(ridge, "ridge regression")
case lasso: LassoModel =>
new GeneralizedLinearPMMLModelExport(lasso, "lasso regression")
case svm: SVMModel =>
new BinaryClassificationPMMLModelExport(
svm, "linear SVM", RegressionNormalizationMethodType.NONE,
svm.getThreshold.getOrElse(0.0))
case logistic: LogisticRegressionModel =>
if (logistic.numClasses == 2) {
new BinaryClassificationPMMLModelExport(
logistic, "logistic regression", RegressionNormalizationMethodType.LOGIT,
logistic.getThreshold.getOrElse(0.5))
} else {
throw new IllegalArgumentException(
"PMML Export not supported for Multinomial Logistic Regression")
}
case _ =>
throw new IllegalArgumentException(
"PMML Export not supported for model: " + model.getClass.getName)
}
}
}
Example 2
| Source File: LinearRegression.scala From Swallow with Apache License 2.0 | 5 votes |
|
package com.intel.hibench.sparkbench.ml
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.LinearRegressionWithSGD
import org.apache.spark.rdd.RDD
import scopt.OptionParser
object LinearRegression {
case class Params(
dataPath: String = null,
numIterations: Int = 100,
stepSize: Double = 0.00000001
)
def main(args: Array[String]): Unit = {
val defaultParams = Params()
val parser = new OptionParser[Params]("Linear"){
head("Linear Regression: an example of linear regression with SGD optimizer")
opt[Int]("numIterations")
.text(s"numIterations, default: ${defaultParams.numIterations}")
.action((x,c) => c.copy(numIterations = x))
opt[Double]("stepSize")
.text(s"stepSize, default: ${defaultParams.stepSize}")
.action((x,c) => c.copy(stepSize = x))
arg[String]("<dataPath>")
.required()
.text("Input path for data")
.action((x,c) => c.copy(dataPath = x))
}
parser.parse(args, defaultParams) match {
case Some(params) => run(params)
case _ => sys.exit(1)
}
}
def run(params: Params): Unit = {
val conf = new SparkConf().setAppName(s"LinearRegressionWithSGD with $params")
.set("spark.shuffle.compress", "false")
.set("spark.io.compression.codec", "org.apache.spark.io.LZFCompressionCodec")
.set("spark.smartCompress", "false")
val sc = new SparkContext(conf)
val dataPath = params.dataPath
val numIterations = params.numIterations
val stepSize = params.stepSize
// Load training data in LabeledPoint format.
val data: RDD[LabeledPoint] = sc.objectFile(dataPath)
// Building the model
val model = LinearRegressionWithSGD.train(data, numIterations, stepSize)
// Evaluate model on training examples and compute training error
val valuesAndPreds = data.map { point =>
val prediction = model.predict(point.features)
(point.label, prediction)
}
val MSE = valuesAndPreds.map{ case(v, p) => math.pow((v - p), 2) }.mean()
println("Training Mean Squared Error = " + MSE)
sc.stop()
}
}
Example 3
| Source File: HogHBaseReputation.scala From hogzilla with GNU General Public License v2.0 | 5 votes |
|
package org.hogzilla.hbase
import scala.math.random
import java.lang.Math
import org.apache.spark._
import org.apache.hadoop.hbase.client.HBaseAdmin
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.{HBaseConfiguration, HTableDescriptor, TableName}
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
import org.apache.spark.mllib.regression.{LabeledPoint,LinearRegressionModel,LinearRegressionWithSGD}
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.rdd.RDD
import org.apache.hadoop.hbase.client.HTable
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter
import org.apache.hadoop.hbase.filter.BinaryComparator
import org.apache.hadoop.hbase.filter.FilterList
import org.apache.hadoop.hbase.filter.CompareFilter
import java.util.ArrayList
import org.apache.hadoop.hbase.client.Scan
import org.apache.hadoop.hbase.filter.Filter
import scala.collection.mutable.HashSet
import org.apache.hadoop.hbase.client.Put
object HogHBaseReputation {
// Ex: MX, whitelist
def getReputationList(listName:String, listType:String):Set[String] =
{
val list = new HashSet[String]
val filters: ArrayList[Filter] = new ArrayList();
val colValFilter1 = new SingleColumnValueFilter(Bytes.toBytes("rep"), Bytes.toBytes("list_type"),
CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(listType)))
colValFilter1.setFilterIfMissing(false);
val colValFilter2 = new SingleColumnValueFilter(Bytes.toBytes("rep"), Bytes.toBytes("list"),
CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(listName)))
colValFilter2.setFilterIfMissing(false);
filters.add(colValFilter1);
filters.add(colValFilter2);
val filterList = new FilterList( FilterList.Operator.MUST_PASS_ALL, filters);
val scan = new Scan()
scan.setFilter(filterList)
val it = HogHBaseRDD.hogzilla_reputation.getScanner(scan).iterator()
while(it.hasNext())
{
list.add( Bytes.toString(it.next().getValue(Bytes.toBytes("rep"),Bytes.toBytes("ip"))) )
}
list.toSet
}
def saveReputationList(listName:String, listType:String, ip:String) =
{
val put = new Put(Bytes.toBytes(ip+"-"+listName+"-"+listType))
put.add(Bytes.toBytes("rep"), Bytes.toBytes("list_type"), Bytes.toBytes(listType))
put.add(Bytes.toBytes("rep"), Bytes.toBytes("list"), Bytes.toBytes(listName))
put.add(Bytes.toBytes("rep"), Bytes.toBytes("ip"), Bytes.toBytes(ip))
HogHBaseRDD.hogzilla_reputation.put(put)
}
}
Example 4
| Source File: PMMLModelExportFactorySuite.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.apache.spark.SparkFunSuite
import org.apache.spark.mllib.classification.{LogisticRegressionModel, SVMModel}
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.{LassoModel, LinearRegressionModel, RidgeRegressionModel}
import org.apache.spark.mllib.util.LinearDataGenerator
class PMMLModelExportFactorySuite extends SparkFunSuite {
test("PMMLModelExportFactory create KMeansPMMLModelExport when passing a KMeansModel") {
val clusterCenters = Array(
Vectors.dense(1.0, 2.0, 6.0),
Vectors.dense(1.0, 3.0, 0.0),
Vectors.dense(1.0, 4.0, 6.0))
val kmeansModel = new KMeansModel(clusterCenters)
val modelExport = PMMLModelExportFactory.createPMMLModelExport(kmeansModel)
assert(modelExport.isInstanceOf[KMeansPMMLModelExport])
}
test("PMMLModelExportFactory create GeneralizedLinearPMMLModelExport when passing a "
+ "LinearRegressionModel, RidgeRegressionModel or LassoModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val linearRegressionModel =
new LinearRegressionModel(linearInput(0).features, linearInput(0).label)
val linearModelExport = PMMLModelExportFactory.createPMMLModelExport(linearRegressionModel)
assert(linearModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val ridgeRegressionModel =
new RidgeRegressionModel(linearInput(0).features, linearInput(0).label)
val ridgeModelExport = PMMLModelExportFactory.createPMMLModelExport(ridgeRegressionModel)
assert(ridgeModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val lassoModel = new LassoModel(linearInput(0).features, linearInput(0).label)
val lassoModelExport = PMMLModelExportFactory.createPMMLModelExport(lassoModel)
assert(lassoModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
}
test("PMMLModelExportFactory create BinaryClassificationPMMLModelExport "
+ "when passing a LogisticRegressionModel or SVMModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val logisticRegressionModel =
new LogisticRegressionModel(linearInput(0).features, linearInput(0).label)
val logisticRegressionModelExport =
PMMLModelExportFactory.createPMMLModelExport(logisticRegressionModel)
assert(logisticRegressionModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
val svmModel = new SVMModel(linearInput(0).features, linearInput(0).label)
val svmModelExport = PMMLModelExportFactory.createPMMLModelExport(svmModel)
assert(svmModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
}
test("PMMLModelExportFactory throw IllegalArgumentException "
+ "when passing a Multinomial Logistic Regression") {
val multiclassLogisticRegressionModel = new LogisticRegressionModel(
weights = Vectors.dense(0.1, 0.2, 0.3, 0.4), intercept = 1.0,
numFeatures = 2, numClasses = 3)
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(multiclassLogisticRegressionModel)
}
}
test("PMMLModelExportFactory throw IllegalArgumentException when passing an unsupported model") {
val invalidModel = new Object
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(invalidModel)
}
}
}
Example 5
| Source File: PMMLModelExportFactory.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.dmg.pmml.RegressionNormalizationMethodType
import org.apache.spark.mllib.classification.LogisticRegressionModel
import org.apache.spark.mllib.classification.SVMModel
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.regression.LassoModel
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.RidgeRegressionModel
private[mllib] object PMMLModelExportFactory {
def createPMMLModelExport(model: Any): PMMLModelExport = {
model match {
case kmeans: KMeansModel =>
new KMeansPMMLModelExport(kmeans)
case linear: LinearRegressionModel =>
new GeneralizedLinearPMMLModelExport(linear, "linear regression")
case ridge: RidgeRegressionModel =>
new GeneralizedLinearPMMLModelExport(ridge, "ridge regression")
case lasso: LassoModel =>
new GeneralizedLinearPMMLModelExport(lasso, "lasso regression")
case svm: SVMModel =>
new BinaryClassificationPMMLModelExport(
svm, "linear SVM", RegressionNormalizationMethodType.NONE,
svm.getThreshold.getOrElse(0.0))
case logistic: LogisticRegressionModel =>
if (logistic.numClasses == 2) {
new BinaryClassificationPMMLModelExport(
logistic, "logistic regression", RegressionNormalizationMethodType.LOGIT,
logistic.getThreshold.getOrElse(0.5))
} else {
throw new IllegalArgumentException(
"PMML Export not supported for Multinomial Logistic Regression")
}
case _ =>
throw new IllegalArgumentException(
"PMML Export not supported for model: " + model.getClass.getName)
}
}
}
Example 6
| Source File: PMMLModelExportFactorySuite.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.apache.spark.SparkFunSuite
import org.apache.spark.mllib.classification.{LogisticRegressionModel, SVMModel}
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.{LassoModel, LinearRegressionModel, RidgeRegressionModel}
import org.apache.spark.mllib.util.LinearDataGenerator
class PMMLModelExportFactorySuite extends SparkFunSuite {
test("PMMLModelExportFactory create KMeansPMMLModelExport when passing a KMeansModel") {
val clusterCenters = Array(
Vectors.dense(1.0, 2.0, 6.0),
Vectors.dense(1.0, 3.0, 0.0),
Vectors.dense(1.0, 4.0, 6.0))
val kmeansModel = new KMeansModel(clusterCenters)
val modelExport = PMMLModelExportFactory.createPMMLModelExport(kmeansModel)
assert(modelExport.isInstanceOf[KMeansPMMLModelExport])
}
test("PMMLModelExportFactory create GeneralizedLinearPMMLModelExport when passing a "
+ "LinearRegressionModel, RidgeRegressionModel or LassoModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val linearRegressionModel =
new LinearRegressionModel(linearInput(0).features, linearInput(0).label)
val linearModelExport = PMMLModelExportFactory.createPMMLModelExport(linearRegressionModel)
assert(linearModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val ridgeRegressionModel =
new RidgeRegressionModel(linearInput(0).features, linearInput(0).label)
val ridgeModelExport = PMMLModelExportFactory.createPMMLModelExport(ridgeRegressionModel)
assert(ridgeModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val lassoModel = new LassoModel(linearInput(0).features, linearInput(0).label)
val lassoModelExport = PMMLModelExportFactory.createPMMLModelExport(lassoModel)
assert(lassoModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
}
test("PMMLModelExportFactory create BinaryClassificationPMMLModelExport "
+ "when passing a LogisticRegressionModel or SVMModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val logisticRegressionModel =
new LogisticRegressionModel(linearInput(0).features, linearInput(0).label)
val logisticRegressionModelExport =
PMMLModelExportFactory.createPMMLModelExport(logisticRegressionModel)
assert(logisticRegressionModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
val svmModel = new SVMModel(linearInput(0).features, linearInput(0).label)
val svmModelExport = PMMLModelExportFactory.createPMMLModelExport(svmModel)
assert(svmModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
}
test("PMMLModelExportFactory throw IllegalArgumentException "
+ "when passing a Multinomial Logistic Regression") {
val multiclassLogisticRegressionModel = new LogisticRegressionModel(
weights = Vectors.dense(0.1, 0.2, 0.3, 0.4), intercept = 1.0,
numFeatures = 2, numClasses = 3)
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(multiclassLogisticRegressionModel)
}
}
test("PMMLModelExportFactory throw IllegalArgumentException when passing an unsupported model") {
val invalidModel = new Object
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(invalidModel)
}
}
}
Example 7
| Source File: PMMLModelExportFactory.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.dmg.pmml.RegressionNormalizationMethodType
import org.apache.spark.mllib.classification.LogisticRegressionModel
import org.apache.spark.mllib.classification.SVMModel
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.regression.LassoModel
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.RidgeRegressionModel
private[mllib] object PMMLModelExportFactory {
def createPMMLModelExport(model: Any): PMMLModelExport = {
model match {
case kmeans: KMeansModel =>
new KMeansPMMLModelExport(kmeans)
case linear: LinearRegressionModel =>
new GeneralizedLinearPMMLModelExport(linear, "linear regression")
case ridge: RidgeRegressionModel =>
new GeneralizedLinearPMMLModelExport(ridge, "ridge regression")
case lasso: LassoModel =>
new GeneralizedLinearPMMLModelExport(lasso, "lasso regression")
case svm: SVMModel =>
new BinaryClassificationPMMLModelExport(
svm, "linear SVM", RegressionNormalizationMethodType.NONE,
svm.getThreshold.getOrElse(0.0))
case logistic: LogisticRegressionModel =>
if (logistic.numClasses == 2) {
new BinaryClassificationPMMLModelExport(
logistic, "logistic regression", RegressionNormalizationMethodType.LOGIT,
logistic.getThreshold.getOrElse(0.5))
} else {
throw new IllegalArgumentException(
"PMML Export not supported for Multinomial Logistic Regression")
}
case _ =>
throw new IllegalArgumentException(
"PMML Export not supported for model: " + model.getClass.getName)
}
}
}
Example 8
| Source File: LinearRegressionWithSGDExample.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
// scalastyle:off println
package org.apache.spark.examples.mllib
import org.apache.spark.{SparkConf, SparkContext}
// $example on$
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.LinearRegressionWithSGD
// $example off$
@deprecated("Use ml.regression.LinearRegression or LBFGS", "2.0.0")
object LinearRegressionWithSGDExample {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("LinearRegressionWithSGDExample")
val sc = new SparkContext(conf)
// $example on$
// Load and parse the data
val data = sc.textFile("data/mllib/ridge-data/lpsa.data")
val parsedData = data.map { line =>
val parts = line.split(',')
LabeledPoint(parts(0).toDouble, Vectors.dense(parts(1).split(' ').map(_.toDouble)))
}.cache()
// Building the model
val numIterations = 100
val stepSize = 0.00000001
val model = LinearRegressionWithSGD.train(parsedData, numIterations, stepSize)
// Evaluate model on training examples and compute training error
val valuesAndPreds = parsedData.map { point =>
val prediction = model.predict(point.features)
(point.label, prediction)
}
val MSE = valuesAndPreds.map{ case(v, p) => math.pow((v - p), 2) }.mean()
println(s"training Mean Squared Error $MSE")
// Save and load model
model.save(sc, "target/tmp/scalaLinearRegressionWithSGDModel")
val sameModel = LinearRegressionModel.load(sc, "target/tmp/scalaLinearRegressionWithSGDModel")
// $example off$
sc.stop()
}
}
// scalastyle:on println
Example 9
| Source File: PMMLModelExportFactorySuite.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.apache.spark.SparkFunSuite
import org.apache.spark.mllib.classification.{LogisticRegressionModel, SVMModel}
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.{LassoModel, LinearRegressionModel, RidgeRegressionModel}
import org.apache.spark.mllib.util.LinearDataGenerator
val multiclassLogisticRegressionModel = new LogisticRegressionModel(
weights = Vectors.dense(0.1, 0.2, 0.3, 0.4), intercept = 1.0,
//numClasses 分类数
numFeatures = 2, numClasses = 3)
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(multiclassLogisticRegressionModel)
}
}
test("PMMLModelExportFactory throw IllegalArgumentException when passing an unsupported model") {
val invalidModel = new Object
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(invalidModel)
}
}
}
Example 10
| Source File: PMMLModelExportFactory.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.dmg.pmml.RegressionNormalizationMethodType
import org.apache.spark.mllib.classification.LogisticRegressionModel
import org.apache.spark.mllib.classification.SVMModel
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.regression.LassoModel
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.RidgeRegressionModel
private[mllib] object PMMLModelExportFactory {
def createPMMLModelExport(model: Any): PMMLModelExport = {
model match {
case kmeans: KMeansModel =>
new KMeansPMMLModelExport(kmeans)
case linear: LinearRegressionModel =>
new GeneralizedLinearPMMLModelExport(linear, "linear regression")
case ridge: RidgeRegressionModel =>
new GeneralizedLinearPMMLModelExport(ridge, "ridge regression")
case lasso: LassoModel =>
new GeneralizedLinearPMMLModelExport(lasso, "lasso regression")
case svm: SVMModel =>
new BinaryClassificationPMMLModelExport(
svm, "linear SVM", RegressionNormalizationMethodType.NONE,
svm.getThreshold.getOrElse(0.0))
case logistic: LogisticRegressionModel =>
if (logistic.numClasses == 2) {
new BinaryClassificationPMMLModelExport(
logistic, "logistic regression", RegressionNormalizationMethodType.LOGIT,
logistic.getThreshold.getOrElse(0.5))
} else {
throw new IllegalArgumentException(
"PMML Export not supported for Multinomial Logistic Regression")
}
case _ =>
throw new IllegalArgumentException(
"PMML Export not supported for model: " + model.getClass.getName)
}
}
}
Example 11
| Source File: PMMLModelExportFactorySuite.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.apache.spark.SparkFunSuite
import org.apache.spark.mllib.classification.{LogisticRegressionModel, SVMModel}
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.{LassoModel, LinearRegressionModel, RidgeRegressionModel}
import org.apache.spark.mllib.util.LinearDataGenerator
class PMMLModelExportFactorySuite extends SparkFunSuite {
test("PMMLModelExportFactory create KMeansPMMLModelExport when passing a KMeansModel") {
val clusterCenters = Array(
Vectors.dense(1.0, 2.0, 6.0),
Vectors.dense(1.0, 3.0, 0.0),
Vectors.dense(1.0, 4.0, 6.0))
val kmeansModel = new KMeansModel(clusterCenters)
val modelExport = PMMLModelExportFactory.createPMMLModelExport(kmeansModel)
assert(modelExport.isInstanceOf[KMeansPMMLModelExport])
}
test("PMMLModelExportFactory create GeneralizedLinearPMMLModelExport when passing a "
+ "LinearRegressionModel, RidgeRegressionModel or LassoModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val linearRegressionModel =
new LinearRegressionModel(linearInput(0).features, linearInput(0).label)
val linearModelExport = PMMLModelExportFactory.createPMMLModelExport(linearRegressionModel)
assert(linearModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val ridgeRegressionModel =
new RidgeRegressionModel(linearInput(0).features, linearInput(0).label)
val ridgeModelExport = PMMLModelExportFactory.createPMMLModelExport(ridgeRegressionModel)
assert(ridgeModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val lassoModel = new LassoModel(linearInput(0).features, linearInput(0).label)
val lassoModelExport = PMMLModelExportFactory.createPMMLModelExport(lassoModel)
assert(lassoModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
}
test("PMMLModelExportFactory create BinaryClassificationPMMLModelExport "
+ "when passing a LogisticRegressionModel or SVMModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val logisticRegressionModel =
new LogisticRegressionModel(linearInput(0).features, linearInput(0).label)
val logisticRegressionModelExport =
PMMLModelExportFactory.createPMMLModelExport(logisticRegressionModel)
assert(logisticRegressionModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
val svmModel = new SVMModel(linearInput(0).features, linearInput(0).label)
val svmModelExport = PMMLModelExportFactory.createPMMLModelExport(svmModel)
assert(svmModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
}
test("PMMLModelExportFactory throw IllegalArgumentException "
+ "when passing a Multinomial Logistic Regression") {
val multiclassLogisticRegressionModel = new LogisticRegressionModel(
weights = Vectors.dense(0.1, 0.2, 0.3, 0.4), intercept = 1.0,
numFeatures = 2, numClasses = 3)
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(multiclassLogisticRegressionModel)
}
}
test("PMMLModelExportFactory throw IllegalArgumentException when passing an unsupported model") {
val invalidModel = new Object
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(invalidModel)
}
}
}
Example 12
| Source File: LinearRegressionWithSGDExample.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
// scalastyle:off println
package org.apache.spark.examples.mllib
import org.apache.spark.{SparkConf, SparkContext}
// $example on$
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.LinearRegressionWithSGD
// $example off$
@deprecated("Use ml.regression.LinearRegression or LBFGS", "2.0.0")
object LinearRegressionWithSGDExample {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("LinearRegressionWithSGDExample")
val sc = new SparkContext(conf)
// $example on$
// Load and parse the data
val data = sc.textFile("data/mllib/ridge-data/lpsa.data")
val parsedData = data.map { line =>
val parts = line.split(',')
LabeledPoint(parts(0).toDouble, Vectors.dense(parts(1).split(' ').map(_.toDouble)))
}.cache()
// Building the model
val numIterations = 100
val stepSize = 0.00000001
val model = LinearRegressionWithSGD.train(parsedData, numIterations, stepSize)
// Evaluate model on training examples and compute training error
val valuesAndPreds = parsedData.map { point =>
val prediction = model.predict(point.features)
(point.label, prediction)
}
val MSE = valuesAndPreds.map{ case(v, p) => math.pow((v - p), 2) }.mean()
println("training Mean Squared Error = " + MSE)
// Save and load model
model.save(sc, "target/tmp/scalaLinearRegressionWithSGDModel")
val sameModel = LinearRegressionModel.load(sc, "target/tmp/scalaLinearRegressionWithSGDModel")
// $example off$
sc.stop()
}
}
// scalastyle:on println
Example 13
| Source File: PMMLModelExportFactorySuite.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.apache.spark.SparkFunSuite
import org.apache.spark.mllib.classification.{LogisticRegressionModel, SVMModel}
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.{LassoModel, LinearRegressionModel, RidgeRegressionModel}
import org.apache.spark.mllib.util.LinearDataGenerator
class PMMLModelExportFactorySuite extends SparkFunSuite {
test("PMMLModelExportFactory create KMeansPMMLModelExport when passing a KMeansModel") {
val clusterCenters = Array(
Vectors.dense(1.0, 2.0, 6.0),
Vectors.dense(1.0, 3.0, 0.0),
Vectors.dense(1.0, 4.0, 6.0))
val kmeansModel = new KMeansModel(clusterCenters)
val modelExport = PMMLModelExportFactory.createPMMLModelExport(kmeansModel)
assert(modelExport.isInstanceOf[KMeansPMMLModelExport])
}
test("PMMLModelExportFactory create GeneralizedLinearPMMLModelExport when passing a "
+ "LinearRegressionModel, RidgeRegressionModel or LassoModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val linearRegressionModel =
new LinearRegressionModel(linearInput(0).features, linearInput(0).label)
val linearModelExport = PMMLModelExportFactory.createPMMLModelExport(linearRegressionModel)
assert(linearModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val ridgeRegressionModel =
new RidgeRegressionModel(linearInput(0).features, linearInput(0).label)
val ridgeModelExport = PMMLModelExportFactory.createPMMLModelExport(ridgeRegressionModel)
assert(ridgeModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val lassoModel = new LassoModel(linearInput(0).features, linearInput(0).label)
val lassoModelExport = PMMLModelExportFactory.createPMMLModelExport(lassoModel)
assert(lassoModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
}
test("PMMLModelExportFactory create BinaryClassificationPMMLModelExport "
+ "when passing a LogisticRegressionModel or SVMModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val logisticRegressionModel =
new LogisticRegressionModel(linearInput(0).features, linearInput(0).label)
val logisticRegressionModelExport =
PMMLModelExportFactory.createPMMLModelExport(logisticRegressionModel)
assert(logisticRegressionModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
val svmModel = new SVMModel(linearInput(0).features, linearInput(0).label)
val svmModelExport = PMMLModelExportFactory.createPMMLModelExport(svmModel)
assert(svmModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
}
test("PMMLModelExportFactory throw IllegalArgumentException "
+ "when passing a Multinomial Logistic Regression") {
val multiclassLogisticRegressionModel = new LogisticRegressionModel(
weights = Vectors.dense(0.1, 0.2, 0.3, 0.4), intercept = 1.0,
numFeatures = 2, numClasses = 3)
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(multiclassLogisticRegressionModel)
}
}
test("PMMLModelExportFactory throw IllegalArgumentException when passing an unsupported model") {
val invalidModel = new Object
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(invalidModel)
}
}
}
Example 14
| Source File: PMMLModelExportFactory.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.dmg.pmml.RegressionNormalizationMethodType
import org.apache.spark.mllib.classification.LogisticRegressionModel
import org.apache.spark.mllib.classification.SVMModel
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.regression.LassoModel
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.RidgeRegressionModel
private[mllib] object PMMLModelExportFactory {
def createPMMLModelExport(model: Any): PMMLModelExport = {
model match {
case kmeans: KMeansModel =>
new KMeansPMMLModelExport(kmeans)
case linear: LinearRegressionModel =>
new GeneralizedLinearPMMLModelExport(linear, "linear regression")
case ridge: RidgeRegressionModel =>
new GeneralizedLinearPMMLModelExport(ridge, "ridge regression")
case lasso: LassoModel =>
new GeneralizedLinearPMMLModelExport(lasso, "lasso regression")
case svm: SVMModel =>
new BinaryClassificationPMMLModelExport(
svm, "linear SVM", RegressionNormalizationMethodType.NONE,
svm.getThreshold.getOrElse(0.0))
case logistic: LogisticRegressionModel =>
if (logistic.numClasses == 2) {
new BinaryClassificationPMMLModelExport(
logistic, "logistic regression", RegressionNormalizationMethodType.LOGIT,
logistic.getThreshold.getOrElse(0.5))
} else {
throw new IllegalArgumentException(
"PMML Export not supported for Multinomial Logistic Regression")
}
case _ =>
throw new IllegalArgumentException(
"PMML Export not supported for model: " + model.getClass.getName)
}
}
}
Example 15
| Source File: LinearRegressionWithSGDExample.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
// scalastyle:off println
package org.apache.spark.examples.mllib
import org.apache.spark.{SparkConf, SparkContext}
// $example on$
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.LinearRegressionWithSGD
// $example off$
@deprecated("Use ml.regression.LinearRegression or LBFGS", "2.0.0")
object LinearRegressionWithSGDExample {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("LinearRegressionWithSGDExample")
val sc = new SparkContext(conf)
// $example on$
// Load and parse the data
val data = sc.textFile("data/mllib/ridge-data/lpsa.data")
val parsedData = data.map { line =>
val parts = line.split(',')
LabeledPoint(parts(0).toDouble, Vectors.dense(parts(1).split(' ').map(_.toDouble)))
}.cache()
// Building the model
val numIterations = 100
val stepSize = 0.00000001
val model = LinearRegressionWithSGD.train(parsedData, numIterations, stepSize)
// Evaluate model on training examples and compute training error
val valuesAndPreds = parsedData.map { point =>
val prediction = model.predict(point.features)
(point.label, prediction)
}
val MSE = valuesAndPreds.map{ case(v, p) => math.pow((v - p), 2) }.mean()
println("training Mean Squared Error = " + MSE)
// Save and load model
model.save(sc, "target/tmp/scalaLinearRegressionWithSGDModel")
val sameModel = LinearRegressionModel.load(sc, "target/tmp/scalaLinearRegressionWithSGDModel")
// $example off$
sc.stop()
}
}
// scalastyle:on println
Example 16
| Source File: LinearRegExample.scala From Hands-On-Data-Analysis-with-Scala with MIT License | 5 votes |
|
package handson.example
import org.apache.spark.sql.SparkSession
object LinearRegExample {
val homeDir = System.getProperty("user.home")
def main(args: Array[String]): Unit = {
// 1. Set Spark session
val spark = SparkSession.builder().master("local").getOrCreate()
// 2. Set logging level to WARNING
spark.sparkContext.setLogLevel("WARN")
// 3. Import necessary classes from Spark MLLib package that are needed for linear regression
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.LinearRegressionWithSGD
// 4. Load the data
val data = spark.sparkContext.textFile(s"${homeDir}/lpsa.data")
// 5. Parse the data into LabeledPoint and cache
val parsedData = data.map { line =>
val parts = line.split(',')
LabeledPoint(parts(0).toDouble, Vectors.dense(parts(1).split(' ').map(_.toDouble)))
}.cache()
// 6. Build the model by setting number of iterations, step size
val numIterations = 100
val stepSize = 0.00000001
val model = LinearRegressionWithSGD.train(parsedData, numIterations, stepSize)
// 7. Evaluate model on training examples and compute training error
val valuesAndPreds = parsedData.map { point =>
val prediction = model.predict(point.features)
(point.label, prediction)
}
val MSE = valuesAndPreds.map{ case(v, p) => math.pow((v - p), 2) }.mean()
println(s"training Mean Squared Error $MSE")
// 8. Save the model
model.save(spark.sparkContext, s"{homeDir}/LinearRegressionWithSGDModel")
// 9. Load the saved model
val sameModel = LinearRegressionModel.load(spark.sparkContext, s"{homeDir}/LinearRegressionWithSGDModel")
// 10. Output the model
println(sameModel)
}
}
Example 17
| Source File: PMMLModelExportFactorySuite.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.apache.spark.SparkFunSuite
import org.apache.spark.mllib.classification.{LogisticRegressionModel, SVMModel}
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.{LassoModel, LinearRegressionModel, RidgeRegressionModel}
import org.apache.spark.mllib.util.LinearDataGenerator
class PMMLModelExportFactorySuite extends SparkFunSuite {
test("PMMLModelExportFactory create KMeansPMMLModelExport when passing a KMeansModel") {
val clusterCenters = Array(
Vectors.dense(1.0, 2.0, 6.0),
Vectors.dense(1.0, 3.0, 0.0),
Vectors.dense(1.0, 4.0, 6.0))
val kmeansModel = new KMeansModel(clusterCenters)
val modelExport = PMMLModelExportFactory.createPMMLModelExport(kmeansModel)
assert(modelExport.isInstanceOf[KMeansPMMLModelExport])
}
test("PMMLModelExportFactory create GeneralizedLinearPMMLModelExport when passing a "
+ "LinearRegressionModel, RidgeRegressionModel or LassoModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val linearRegressionModel =
new LinearRegressionModel(linearInput(0).features, linearInput(0).label)
val linearModelExport = PMMLModelExportFactory.createPMMLModelExport(linearRegressionModel)
assert(linearModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val ridgeRegressionModel =
new RidgeRegressionModel(linearInput(0).features, linearInput(0).label)
val ridgeModelExport = PMMLModelExportFactory.createPMMLModelExport(ridgeRegressionModel)
assert(ridgeModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val lassoModel = new LassoModel(linearInput(0).features, linearInput(0).label)
val lassoModelExport = PMMLModelExportFactory.createPMMLModelExport(lassoModel)
assert(lassoModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
}
test("PMMLModelExportFactory create BinaryClassificationPMMLModelExport "
+ "when passing a LogisticRegressionModel or SVMModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val logisticRegressionModel =
new LogisticRegressionModel(linearInput(0).features, linearInput(0).label)
val logisticRegressionModelExport =
PMMLModelExportFactory.createPMMLModelExport(logisticRegressionModel)
assert(logisticRegressionModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
val svmModel = new SVMModel(linearInput(0).features, linearInput(0).label)
val svmModelExport = PMMLModelExportFactory.createPMMLModelExport(svmModel)
assert(svmModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
}
test("PMMLModelExportFactory throw IllegalArgumentException "
+ "when passing a Multinomial Logistic Regression") {
val multiclassLogisticRegressionModel = new LogisticRegressionModel(
weights = Vectors.dense(0.1, 0.2, 0.3, 0.4), intercept = 1.0,
numFeatures = 2, numClasses = 3)
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(multiclassLogisticRegressionModel)
}
}
test("PMMLModelExportFactory throw IllegalArgumentException when passing an unsupported model") {
val invalidModel = new Object
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(invalidModel)
}
}
}
Example 18
| Source File: PMMLModelExportFactory.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.dmg.pmml.RegressionNormalizationMethodType
import org.apache.spark.mllib.classification.LogisticRegressionModel
import org.apache.spark.mllib.classification.SVMModel
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.regression.LassoModel
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.RidgeRegressionModel
private[mllib] object PMMLModelExportFactory {
def createPMMLModelExport(model: Any): PMMLModelExport = {
model match {
case kmeans: KMeansModel =>
new KMeansPMMLModelExport(kmeans)
case linear: LinearRegressionModel =>
new GeneralizedLinearPMMLModelExport(linear, "linear regression")
case ridge: RidgeRegressionModel =>
new GeneralizedLinearPMMLModelExport(ridge, "ridge regression")
case lasso: LassoModel =>
new GeneralizedLinearPMMLModelExport(lasso, "lasso regression")
case svm: SVMModel =>
new BinaryClassificationPMMLModelExport(
svm, "linear SVM", RegressionNormalizationMethodType.NONE,
svm.getThreshold.getOrElse(0.0))
case logistic: LogisticRegressionModel =>
if (logistic.numClasses == 2) {
new BinaryClassificationPMMLModelExport(
logistic, "logistic regression", RegressionNormalizationMethodType.LOGIT,
logistic.getThreshold.getOrElse(0.5))
} else {
throw new IllegalArgumentException(
"PMML Export not supported for Multinomial Logistic Regression")
}
case _ =>
throw new IllegalArgumentException(
"PMML Export not supported for model: " + model.getClass.getName)
}
}
}
Example 19
| Source File: LinearRegressionWithSGDExample.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
// scalastyle:off println
package org.apache.spark.examples.mllib
import org.apache.spark.{SparkConf, SparkContext}
// $example on$
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.LinearRegressionWithSGD
// $example off$
@deprecated("Use ml.regression.LinearRegression or LBFGS", "2.0.0")
object LinearRegressionWithSGDExample {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("LinearRegressionWithSGDExample")
val sc = new SparkContext(conf)
// $example on$
// Load and parse the data
val data = sc.textFile("data/mllib/ridge-data/lpsa.data")
val parsedData = data.map { line =>
val parts = line.split(',')
LabeledPoint(parts(0).toDouble, Vectors.dense(parts(1).split(' ').map(_.toDouble)))
}.cache()
// Building the model
val numIterations = 100
val stepSize = 0.00000001
val model = LinearRegressionWithSGD.train(parsedData, numIterations, stepSize)
// Evaluate model on training examples and compute training error
val valuesAndPreds = parsedData.map { point =>
val prediction = model.predict(point.features)
(point.label, prediction)
}
val MSE = valuesAndPreds.map{ case(v, p) => math.pow((v - p), 2) }.mean()
println("training Mean Squared Error = " + MSE)
// Save and load model
model.save(sc, "target/tmp/scalaLinearRegressionWithSGDModel")
val sameModel = LinearRegressionModel.load(sc, "target/tmp/scalaLinearRegressionWithSGDModel")
// $example off$
sc.stop()
}
}
// scalastyle:on println
Example 20
| Source File: PMMLModelExportFactorySuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.apache.spark.SparkFunSuite
import org.apache.spark.mllib.classification.{LogisticRegressionModel, SVMModel}
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.{LassoModel, LinearRegressionModel, RidgeRegressionModel}
import org.apache.spark.mllib.util.LinearDataGenerator
class PMMLModelExportFactorySuite extends SparkFunSuite {
test("PMMLModelExportFactory create KMeansPMMLModelExport when passing a KMeansModel") {
val clusterCenters = Array(
Vectors.dense(1.0, 2.0, 6.0),
Vectors.dense(1.0, 3.0, 0.0),
Vectors.dense(1.0, 4.0, 6.0))
val kmeansModel = new KMeansModel(clusterCenters)
val modelExport = PMMLModelExportFactory.createPMMLModelExport(kmeansModel)
assert(modelExport.isInstanceOf[KMeansPMMLModelExport])
}
test("PMMLModelExportFactory create GeneralizedLinearPMMLModelExport when passing a "
+ "LinearRegressionModel, RidgeRegressionModel or LassoModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val linearRegressionModel =
new LinearRegressionModel(linearInput(0).features, linearInput(0).label)
val linearModelExport = PMMLModelExportFactory.createPMMLModelExport(linearRegressionModel)
assert(linearModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val ridgeRegressionModel =
new RidgeRegressionModel(linearInput(0).features, linearInput(0).label)
val ridgeModelExport = PMMLModelExportFactory.createPMMLModelExport(ridgeRegressionModel)
assert(ridgeModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
val lassoModel = new LassoModel(linearInput(0).features, linearInput(0).label)
val lassoModelExport = PMMLModelExportFactory.createPMMLModelExport(lassoModel)
assert(lassoModelExport.isInstanceOf[GeneralizedLinearPMMLModelExport])
}
test("PMMLModelExportFactory create BinaryClassificationPMMLModelExport "
+ "when passing a LogisticRegressionModel or SVMModel") {
val linearInput = LinearDataGenerator.generateLinearInput(3.0, Array(10.0, 10.0), 1, 17)
val logisticRegressionModel =
new LogisticRegressionModel(linearInput(0).features, linearInput(0).label)
val logisticRegressionModelExport =
PMMLModelExportFactory.createPMMLModelExport(logisticRegressionModel)
assert(logisticRegressionModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
val svmModel = new SVMModel(linearInput(0).features, linearInput(0).label)
val svmModelExport = PMMLModelExportFactory.createPMMLModelExport(svmModel)
assert(svmModelExport.isInstanceOf[BinaryClassificationPMMLModelExport])
}
test("PMMLModelExportFactory throw IllegalArgumentException "
+ "when passing a Multinomial Logistic Regression") {
val multiclassLogisticRegressionModel = new LogisticRegressionModel(
weights = Vectors.dense(0.1, 0.2, 0.3, 0.4), intercept = 1.0,
numFeatures = 2, numClasses = 3)
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(multiclassLogisticRegressionModel)
}
}
test("PMMLModelExportFactory throw IllegalArgumentException when passing an unsupported model") {
val invalidModel = new Object
intercept[IllegalArgumentException] {
PMMLModelExportFactory.createPMMLModelExport(invalidModel)
}
}
}
Example 21
| Source File: PMMLModelExportFactory.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.mllib.pmml.export
import org.dmg.pmml.RegressionNormalizationMethodType
import org.apache.spark.mllib.classification.LogisticRegressionModel
import org.apache.spark.mllib.classification.SVMModel
import org.apache.spark.mllib.clustering.KMeansModel
import org.apache.spark.mllib.regression.LassoModel
import org.apache.spark.mllib.regression.LinearRegressionModel
import org.apache.spark.mllib.regression.RidgeRegressionModel
private[mllib] object PMMLModelExportFactory {
def createPMMLModelExport(model: Any): PMMLModelExport = {
model match {
case kmeans: KMeansModel =>
new KMeansPMMLModelExport(kmeans)
case linear: LinearRegressionModel =>
new GeneralizedLinearPMMLModelExport(linear, "linear regression")
case ridge: RidgeRegressionModel =>
new GeneralizedLinearPMMLModelExport(ridge, "ridge regression")
case lasso: LassoModel =>
new GeneralizedLinearPMMLModelExport(lasso, "lasso regression")
case svm: SVMModel =>
new BinaryClassificationPMMLModelExport(
svm, "linear SVM", RegressionNormalizationMethodType.NONE,
svm.getThreshold.getOrElse(0.0))
case logistic: LogisticRegressionModel =>
if (logistic.numClasses == 2) {
new BinaryClassificationPMMLModelExport(
logistic, "logistic regression", RegressionNormalizationMethodType.LOGIT,
logistic.getThreshold.getOrElse(0.5))
} else {
throw new IllegalArgumentException(
"PMML Export not supported for Multinomial Logistic Regression")
}
case _ =>
throw new IllegalArgumentException(
"PMML Export not supported for model: " + model.getClass.getName)
}
}
}
