2019獨角獸企業重金招聘Python工程師標準>>>

Spark SQL 最簡單例子 博客分類： spark ? <dependency><groupId>org.apache.spark</groupId><artifactId>spark-core_2.10</artifactId><version>2.1.0</version></dependency><dependency><groupId>org.apache.spark</groupId><artifactId>spark-sql_2.10</artifactId><version>2.1.0</version></dependency>

? ?

??

import java.io.Serializable; import java.util.Arrays;import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SQLContext; import org.apache.spark.sql.SparkSession;public class SparkSqlTest {public static class Person implements Serializable {private static final long serialVersionUID = -6259413972682177507L;private String name;private int age;public Person(String name, int age) {this.name = name;this.age = age;}public String toString() {return name + ": " + age;}public String getName() {return name;}public void setName(String name) {this.name = name;}public int getAge() {return age;}public void setAge(int age) {this.age = age;}}public static void main(String[] args) {SparkConf conf = new SparkConf().setAppName("Test").setMaster("local");JavaSparkContext sc = new JavaSparkContext(conf);SparkSession spark = SparkSession.builder().appName("Test").getOrCreate();JavaRDD<String> input = sc.parallelize(Arrays.asList("abc,1", "test,2"));JavaRDD<Person> persons = input.map(s -> s.split(",")).map(s -> new Person(s[0], Integer.parseInt(s[1])));//[abc: 1, test: 2]System.out.println(persons.collect());Dataset<Row> df = spark.createDataFrame(persons, Person.class);/*+---+----+|age|name|+---+----+| 1| abc|| 2|test|+---+----+*/df.show();/*root|-- age: integer (nullable = false)|-- name: string (nullable = true)*/df.printSchema();SQLContext sql = new SQLContext(spark);sql.registerDataFrameAsTable(df, "person");/*+---+----+|age|name|+---+----+| 2|test|+---+----+*/sql.sql("SELECT * FROM person WHERE age>1").show();sc.close();}}

?

轉載于:https://my.oschina.net/xiaominmin/blog/1598473

總結

以上是生活随笔為你收集整理的Spark SQL 最简单例子的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。