Differences

This shows you the differences between two versions of the page.

--- en:site:recherche:logiciels:sparqlwithspark:watdivc3 [15/09/2016 09:30] – created hubert
+++ en:site:recherche:logiciels:sparqlwithspark:watdivc3 [16/09/2016 23:09] (current) – [S2RDF+Hybrid plan] hubert
@@ Line 1: / Line 1: @@
-====== WatDiv Query C3 ======
+{{indexmenu_n>4}}
+====== WatDiv Query C3 plans======
+===== S2RDF plan =====
+<code scala>
+val c3AllProp = List(
+  ("wsdbm", "likes"),
+  ("wsdbm", "friendOf"),
+  ("dc", "Location"),
+  ("foaf", "age"),
+  ("wsdbm", "gender"),
+  ("foaf", "givenName"))
+//persist all C3 VPs
+c3AllProp.map{case(ns, p) => VP2Random(getIdP(ns, p)).persist().count}
+// Order by increasing tp size
+val order = c3AllProp.map{ case(ns,p) => (p, getIdP(ns,p), VP2Size.get(getIdP(ns,p)).get)}.sortBy{case (p, idp, s)=> s}
+order.foreach(println)
+/*
+(Location,73,4000805)
+(age,83,5001999)
+(gender,14,6000117)
+(givenName,13,7000273)
+(likes,15,11210407)
+(friendOf,17,450067461)
+ */
+val VP2EXP=VP2Random
+// join
+val orderedList = List(
+  ("dc", "Location"),
+  ("foaf", "age"),
+  ("wsdbm", "gender"),
+  ("foaf", "givenName"),
+  ("wsdbm", "likes"),
+  ("wsdbm", "friendOf"))
+val l1 = orderedList.map{case(ns, p) => {
+  val idP = getIdP(ns, p)
+  VP2EXP(idP).withColumnRenamed("o", s"o$idP")}}
+val c3= l1(0).join(l1(1),"s").join(l1(2),"s").join(l1(3),"s").join(l1(4),"s").join(l1(5),"s")
+//c3.count
+//42 845 342
+queryTimeDFIter(c3, 10)
+//time=13.8 s SHFR=1.7GB input=5.2GB
+</code>
+==== S2RDF+Hybrid plan ====
+<code scala>
+val c3AllProp = List(
+  ("wsdbm", "likes"),
+  ("wsdbm", "friendOf"),
+  ("dc", "Location"),
+  ("foaf", "age"),
+  ("wsdbm", "gender"),
+  ("foaf", "givenName"))
+//persist all C3 VPs
+c3AllProp.map{case(ns, p) => VP2(getIdP(ns, p)).persist().count}
+// sort by increasing tp size
+val order = c3AllProp.map{ case(ns,p) => (p, getIdP(ns,p), VP2Size.get(getIdP(ns,p)).get)}.sortBy{case (p, idp, s)=> s}
+/*
+order.foreach(println)
+(Location,73,4000805)
+(age,83,5001999)
+(gender,14,6000117)
+(givenName,13,7000273)
+(likes,15,11210407)
+(friendOf,17,450067461)
+ */
+// VP's are partitioned by SUBJECT
+val VP2EXP=VP2
+// join
+val orderedList = List(
+  ("dc", "Location"),
+  ("foaf", "age"),
+  ("wsdbm", "gender"),
+  ("foaf", "givenName"),
+  ("wsdbm", "likes"),
+  ("wsdbm", "friendOf"))
+val l1 = orderedList.map{case(ns, p) => {
+  val idP = getIdP(ns, p)
+  VP2EXP(idP).withColumnRenamed("o", s"o$idP")}}
+val c3= l1(0).join(l1(1),"s").join(l1(2),"s").join(l1(3),"s").join(l1(4),"s").join(l1(5),"s")
+//c3.count
+//42 845 342
+queryTimeDFIter(c3, 10)
+//time=6,4s  shfr=0  input=257MB
+</code>
 Go to [[en:site:recherche:logiciels:sparqlwithspark]]