Spark SQL怎么用

这篇文章主要介绍“spark sql怎么用”，在日常操作中，相信很多人在Spark SQL怎么用问题上存在疑惑，小编查阅了各式资料，整理出简单好用的操作方法，希望对大家解答”Spark SQL怎么用”的疑惑有所帮助！接下来，请跟着小编一起来学习吧！

pom.xml
<dependencies>

1.     <dependency>
   

2.         <groupId>org.apache.spark</groupId>
   

3.         <artifactId>spark-core_2.10</artifactId>
   

4.         <version>2.1.0</version>
   

5.     </dependency>
   

6.     <dependency>
   

7.         <groupId>org.apache.spark</groupId>
   

8.         <artifactId>spark-sql_2.10</artifactId>
   

9.         <version>2.1.0</version>
   

10.     </dependency>
    

11. </dependencies>

Java:

1. import java.io.Serializable;
   

2. import java.util.Arrays;
   

4. import org.apache.spark.SparkConf;
   

5. import org.apache.spark.api.java.JavaRDD;
   

6. import org.apache.spark.api.java.JavaSparkContext;
   

7. import org.apache.spark.sql.Dataset;
   

8. import org.apache.spark.sql.Row;
   

9. import org.apache.spark.sql.SQLContext;
   

10. import org.apache.spark.sql.SparkSession;
    

12. public class SparkSqlTest {
    

13.     public static class Person implements Serializable {
    

14.         private static final long serialVersionUID = -6259413972682177507L;
    

15.         private String name;
    

16.         private int age;
    

17.         
    

18.         public Person(String name, int age) {
    

19.             this.name = name;
    

20.             this.age = age;
    

21.         }
    

22.         public String toString() {
    

23.             return name + ": " + age;
    

24.         }
    

25.         public String getName() {
    

26.             return name;
    

27.         }
    

28.         public void setName(String name) {
    

29.             this.name = name;
    

30.         }
    

31.         public int getAge() {
    

32.             return age;
    

33.         }
    

34.         public void setAge(int age) {
    

35.             this.age = age;
    

36.         }
    

37.     }
    

38.     
    

39.     public static void main(String[] args) {
    

40.         SparkConf conf = new SparkConf().setAppName("Test").setMaster("local");
    

41.         JavaSparkContext sc = new JavaSparkContext(conf);
    

42.         
    

43.         SparkSession spark = SparkSession.builder().appName("Test").getOrCreate();
    

44.         JavaRDD<String> input = sc.parallelize(Arrays.asList("abc,1", "test,2"));
    

45.         JavaRDD<Person> persons = input.map(s -> s.split(",")).map(s -> new Person(s[0], Integer.parseInt(s[1])));
    

46.         

47.         //[abc: 1, test: 2]

48.         System.out.println(persons.collect());
    

49.         
    

50.         Dataset<Row> df = spark.createDataFrame(persons, Person.class);
    

51.         
    

52.         
    

53.         df.show();
    

54.         
    

55.         
    

56.         df.printSchema();
    

58.         SQLContext sql = new SQLContext(spark);
    

59.         sql.reGISterDataFrameAsTable(df, "person");
    

60.         
    

61.         
    

62.         sql.sql("SELECT * FROM person WHERE age>1").show();
    

63.         
    

64.         sc.close();
    

65.     }
    

67. }

到此，关于“Spark SQL怎么用”的学习就结束了，希望能够解决大家的疑惑。理论与实践的搭配能更好的帮助大家学习，快去试试吧！若想继续学习更多相关知识，请继续关注编程网网站，小编会继续努力为大家带来更多实用的文章！