Ceci est une ancienne révision du document !
Table des matières
Recueil d'examens
Partie MR et Spark
Novembre 2017
Exercice 2 : Algèbre Dataset de Spark
Requêtes
<code> //preparation case class Triple(sujet: String, prop: String, objet: String) val triples = sc.textFile(someFile).
map(ligne => ligne.split(",")).
map(tab => Triple(tab(0), tab(1), tab(2))).toDS()
val t1 = triples.withColumnRenamed(“sujet”,“x”).withColumnRenamed(“objet”,“y”).select(“x”,“y”)
val t2 = triples.withColumnRenamed(“sujet”,“y”).withColumnRenamed(“objet”,“z”).select(“y”,“z”)
val t3 = triples.withColumnRenamed(“sujet”,“z”).withColumnRenamed(“objet”,“x1”).select(“z”,“x1”)
val res = t1.join(t2, “y”).join(t3, “z”).where(“x=x1”)
val t_types = triples.where(“prop='isa'”).select(“sujet”, “objet”)
val t_m = triples.where(“prop!='isa'”).withColumnRenamed(“objet”, “mp”).withColumnRenamed(“sujet”,“m”)
val t_n = triples.where(“prop!='isa'”).withColumnRenamed(“objet”, “np”).withColumnRenamed(“sujet”,“n”)
val t_res = t_m.join(t_n, “prop”).where(“n<m”). join(t_types.withColumnRenamed(“sujet”, “m”), “m”). join(t_types.withColumnRenamed(“sujet”,“n”).withColumnRenamed(“objet”,“o”),“n”).where(“objet=o”).select(“prop”, “np”,“mp”). join(t_types.withColumnRenamed(“sujet”,“mp”), “mp”). join(t_types.withColumnRenamed(“sujet”,“np”).withColumnRenamed(“objet”, “o”),“np”).where(“objet!=o”) </scala>
