spark

Type Members

class AddFields extends RxFrameTransformer

Adds extra columns to the DataFrame.
class BasicStats extends RxFrameTransformer

Calculates basic statistics.
class Cache extends RxFrameTransformer

Caches each DataFrame that arrives to this block.
class CassandraInput extends AbstractRxBlock[DataFrame]

Reads rows from an Apache Cassandra table.
class CassandraOutput extends RxFrameTransformer

Writes rows into a Cassandra table.
class ColumnStats extends RxFrameTransformer

Calculates column-based statistics using MLLib library.
class Correlation extends RxFrameTransformer

Computes the correlation between data series using MLLib library.
class CsvFileInput extends AbstractRxBlock[DataFrame]

Reads a CSV file and generates a spark data frame.
class DataGrid extends AbstractRxBlock[DataFrame]

Creates data frames from static data grid.
class DebugOutput extends RxFrameTransformer

Prints out the data frame data to the standard output.
class Filter extends RxFrameTransformer

Filters the data frame based on a combination of boolean conditions against fields.
class Formula extends RxFrameTransformer

Calculates new fields based on string expressions in various dialects.
class Intersection extends RxMergerN[DataFrame, DataFrame]

Finds the intersection of the two DataRow RDDs.
Finds the intersection of the two DataRow RDDs. They must have idential metadata.
class JdbcInput extends AbstractRxBlock[DataFrame]

Reads data from a JDBC source.
class JdbcOutput extends RxFrameTransformer

Writes data into a database table over JDBC.
class Join extends RxMerger2[DataFrame, DataFrame, DataFrame]

Performs join of the two data frames.
Performs join of the two data frames. In row conditions, if there is ambiguity in a field's name, use "input0" and "input1" prefixes for the first and second input respectively.
class JsonFileInput extends AbstractRxBlock[DataFrame]

Reads a JSON file, which contains a separate JSON object in each line.
class MongoInput extends AbstractRxBlock[DataFrame]

Reads documents from MongoDB.
class MongoOutput extends RxFrameTransformer

Writes documents into a MongoDB collection.
class Reduce extends RxFrameTransformer

Performs reduceByKey() function by grouping the rows by the selected key first, and then applying a list of reduce functions to the specified data columns.
class Regression extends RxFrameTransformer

Computes the regression using MLLib library.
class Repartition extends RxFrameTransformer

Repartitions the underlying Spark RDD.
class RestClient extends RxFrameTransformer

HTTP REST Client, executes one request per row.
abstract class RxFrameTransformer extends RxTransformer[DataFrame, DataFrame]

RX Transformer built on top of an Ignition FrameTransformer class, works as a bridge between ignition transformers and RX transformers.
class SQLQuery extends RxMergerN[DataFrame, DataFrame]

Executes an SQL statement against the inputs.
Executes an SQL statement against the inputs. Each input is injected as a table under the name "inputX" where X is the index of the input.
class SelectValues extends RxFrameTransformer

Modifies, deletes, retains columns in the data rows.
class SetVariables extends RxFrameTransformer

Sets or drops the ignition runtime variables.
class TextFileInput extends AbstractRxBlock[DataFrame]

Reads the text file into a data frame with a single column.
Reads the text file into a data frame with a single column. If the separator is specified, splits the file into multiple rows, otherwise the data frame will contain only one row.
class TextFileOutput extends RxFrameTransformer

Writes rows to a CSV file.
class TextFolderInput extends AbstractRxBlock[DataFrame]

Reads a folder of text files.
class Union extends RxMergerN[DataFrame, DataFrame]

Merges multiple DataFrames.
Merges multiple DataFrames. All of them must have identical schema.

Value Members

object AddFields

Factory for AddFields instances.
object BasicStats

Factory for BasicStats instances.
object Cache

Factory for Cache instances.
object CassandraInput

Factory for CassandraInput instances.
object CassandraOutput

Factory for CassandraOutput instances.
object ColumnStats

Factory for ColumnStats instances.
object Correlation

Factory for Correlation instances.
object CsvFileInput

Factory for CsvFileInput instances.
val DATA_TYPE: String
object DataGrid

Factory for DataGrid instances.
object DebugOutput

Factory for DebugOutput instances.
object Filter

Factory for Filter instances.
object Formula

Factory for Formula instances.
object Intersection

Factory for Intersection instances.
object JdbcInput

Factory for JdbcInput instances.
object JdbcOutput

Factory for JdbcOutput instances.
object Join

Factory for Join instances.
object JsonFileInput

Factory for JsonFileInput instances.
object MongoInput

Factory for MongoInput instances.
object MongoOutput

Factory for MongoOutput instances.
object Reduce

Factory for Reduce instances.
object Regression

Factory for Regression instances.
object Repartition

Factory for Repartition instances.
object RestClient

Factory for RestClient instances.
object SQLQuery

Factory for SQLQuery instances.
object SelectValues

Factory for SelectValues instance.
object SetVariables

Factory for SetVariables instances.
object SparkBlockFactory extends TypeConverters

Spark RX blocks.
object TextFileInput

Factory for TextFileInput instances.
object TextFileOutput

Factory for TextFileOutput instances.
object TextFolderInput

Factory for TextFolderInput instances.
object Union

Factory for Union instances.
def dataFrameToTableData(df: DataFrame): Map[String, Any]

Converts a Spark DataFrame into a tabledata Map.
val dataTypes: List[AtomicType with Product with Serializable]
def extractSeparatedStrings(separator: String)(json: JsonObject, key: String): List[String]

Extracts a string from JSON and splits into a list of strings using the separator.
val extractSeparatedStrings: (JsonObject, String) ⇒ List[String]

Extracts a string from JSON and splits into a list of strings using the separator.
def extractStructFields(withNullable: Boolean)(json: JsonObject, key: String): List[StructField]

Extracts a list of StructFields elements from JSON.
def parseValue(str: String, typeName: String): Any

Parses the string value using the supplied type name.
def producer(df: DataFrame): FrameProducer

Creates a producer, wrapper for the data frame.
def rddToTableData(rdd: RDD[Row], schema: Option[StructType] = None): Map[String, Any]

Converts an RDD[Row] into a tabledata Map.

package spark

Type Members

class AddFields extends RxFrameTransformer

class BasicStats extends RxFrameTransformer

class Cache extends RxFrameTransformer

class CassandraInput extends AbstractRxBlock[DataFrame]

class CassandraOutput extends RxFrameTransformer

class ColumnStats extends RxFrameTransformer

class Correlation extends RxFrameTransformer

class CsvFileInput extends AbstractRxBlock[DataFrame]

class DataGrid extends AbstractRxBlock[DataFrame]

class DebugOutput extends RxFrameTransformer

class Filter extends RxFrameTransformer

class Formula extends RxFrameTransformer

class Intersection extends RxMergerN[DataFrame, DataFrame]

class JdbcInput extends AbstractRxBlock[DataFrame]

class JdbcOutput extends RxFrameTransformer

class Join extends RxMerger2[DataFrame, DataFrame, DataFrame]

class JsonFileInput extends AbstractRxBlock[DataFrame]

class MongoInput extends AbstractRxBlock[DataFrame]

class MongoOutput extends RxFrameTransformer

class Reduce extends RxFrameTransformer

class Regression extends RxFrameTransformer

class Repartition extends RxFrameTransformer

class RestClient extends RxFrameTransformer

abstract class RxFrameTransformer extends RxTransformer[DataFrame, DataFrame]

class SQLQuery extends RxMergerN[DataFrame, DataFrame]

class SelectValues extends RxFrameTransformer

class SetVariables extends RxFrameTransformer

class TextFileInput extends AbstractRxBlock[DataFrame]

class TextFileOutput extends RxFrameTransformer

class TextFolderInput extends AbstractRxBlock[DataFrame]

class Union extends RxMergerN[DataFrame, DataFrame]

Value Members

object AddFields

object BasicStats

object Cache

object CassandraInput

object CassandraOutput

object ColumnStats

object Correlation

object CsvFileInput

val DATA_TYPE: String

object DataGrid

object DebugOutput

object Filter

object Formula

object Intersection

object JdbcInput

object JdbcOutput

object Join

object JsonFileInput

object MongoInput

object MongoOutput

object Reduce

object Regression

object Repartition

object RestClient

object SQLQuery

object SelectValues

object SetVariables

object SparkBlockFactory extends TypeConverters

object TextFileInput

object TextFileOutput

object TextFolderInput

object Union

def dataFrameToTableData(df: DataFrame): Map[String, Any]

val dataTypes: List[AtomicType with Product with Serializable]

def extractSeparatedStrings(separator: String)(json: JsonObject, key: String): List[String]

val extractSeparatedStrings: (JsonObject, String) ⇒ List[String]

def extractStructFields(withNullable: Boolean)(json: JsonObject, key: String): List[StructField]

def parseValue(str: String, typeName: String): Any

def producer(df: DataFrame): FrameProducer

def rddToTableData(rdd: RDD[Row], schema: Option[StructType] = None): Map[String, Any]

Inherited from AnyRef

Inherited from Any

Ungrouped