site:recherche:logiciels:rdfdist
Différences
Ci-dessous, les différences entre deux révisions de la page.
| Les deux révisions précédentesRévision précédenteProchaine révision | Révision précédente | ||
| site:recherche:logiciels:rdfdist [11/05/2015 14:01] – amine | site:recherche:logiciels:rdfdist [26/04/2017 13:06] (Version actuelle) – amann | ||
|---|---|---|---|
| Ligne 1: | Ligne 1: | ||
| - | ===== RDFdist ===== | + | {{indexmenu_n> |
| + | |||
| + | ===== RDFdist | ||
| This wiki page provides information about the experiments RDF distribution approaches using [[http:// | This wiki page provides information about the experiments RDF distribution approaches using [[http:// | ||
| Ligne 11: | Ligne 13: | ||
| We also created two queries for the [[https:// | We also created two queries for the [[https:// | ||
| - | ===Query 1 (synthetic, | + | ===Query 1 (synthetic, |
| < | < | ||
| SELECT ?x ?y ?z | SELECT ?x ?y ?z | ||
| Ligne 108: | Ligne 110: | ||
| val t1 = java.lang.System.currentTimeMillis(); | val t1 = java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | |||
| + | /** | ||
| + | * set inputData with the path to the data encoded as quadruples (see Datasets excerpts) | ||
| + | */ | ||
| // loading and transformating the dataset | // loading and transformating the dataset | ||
| Ligne 149: | Ligne 156: | ||
| val takco : Long = 1115684864 | val takco : Long = 1115684864 | ||
| - | def ajout(a : ListBuffer[(Long, | ||
| - | a += e | ||
| - | return a | ||
| - | } | ||
| // ----------------------------------------------------------- | // ----------------------------------------------------------- | ||
| Ligne 260: | Ligne 263: | ||
| ====Graph partitioning-based approaches==== | ====Graph partitioning-based approaches==== | ||
| + | ===Huang Approach === | ||
| + | < | ||
| + | import org.apache.spark.HashPartitioner | ||
| + | import scala.collection.mutable.ListBuffer | ||
| + | |||
| + | val folder= | ||
| + | val dataset= " | ||
| + | val scale=" | ||
| + | val part=20 //10, 20 | ||
| + | |||
| + | val folderName = folder +scale | ||
| + | val fileName = dataset+scale+" | ||
| + | |||
| + | val t1 = java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | val quads_I_SPO = sc.textFile(s"/ | ||
| + | |||
| + | val quadsDist = quads_I_SPO.partitionBy(new HashPartitioner(part)).persist | ||
| + | |||
| + | |||
| + | |||
| + | val t2 = java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | print(" | ||
| + | |||
| + | val advisor : Long = 1233125376 | ||
| + | val worksFor : Long = 1136656384 | ||
| + | val suborg : Long = 1224736768 | ||
| + | val memof : Long = 113246208 | ||
| + | val undeg : Long = 1101004800 | ||
| + | val teaof : Long = 1199570944 | ||
| + | val takco : Long = 1115684864 | ||
| + | |||
| + | |||
| + | // ----------------------------------------------------------- | ||
| + | // Query 1 : (not part of the benchmark) | ||
| + | // Pattern: (x advisor y) (y worksFor z) (z subOrganisation t) | ||
| + | // ----------------------------------------------------------- | ||
| + | |||
| + | var t1 = java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | var pataws = quadsDist.filter({case(i, | ||
| + | join(quadsDist.filter({case(i, | ||
| + | map({case (y,(x,z)) => (z, | ||
| + | join(quadsDist.filter({case(i, | ||
| + | map({case (z, | ||
| + | |||
| + | pataws.count | ||
| + | |||
| + | var pataws2 = pataws.flatMap(x=> | ||
| + | |||
| + | var t2= java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | println(" | ||
| + | |||
| + | |||
| + | // ----------------------------------------------------------- | ||
| + | // LUBM 2 : MSU | ||
| + | // Pattern: (x memberOf y) (y subOrg z) (x UndergraduateDegreeFrom z) | ||
| + | // ----------------------------------------------------------- | ||
| + | |||
| + | var t1 = java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | //var pmemof = quadsDist.filter({case(i, | ||
| + | |||
| + | var patmsu = quadsDist.filter({case(i, | ||
| + | | ||
| + | | ||
| + | | ||
| + | |||
| + | patmsu.count | ||
| + | |||
| + | var patmsu2 = patmsu.flatMap(identity).distinct | ||
| + | |||
| + | var t2= java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | println(" | ||
| + | |||
| + | // ----------------------------------------------------------- | ||
| + | // LUBM 9 : ATT | ||
| + | // Pattern: (x advisor y) (y teacherOf z) (x takesCourse z) | ||
| + | // ----------------------------------------------------------- | ||
| + | |||
| + | var t1 = java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | var patatt = quadsDist.filter({case(i, | ||
| + | join(quadsDist.filter({case(i, | ||
| + | map({case (y,(x,z)) => (x+"" | ||
| + | join(quadsDist.filter({case(i, | ||
| + | |||
| + | patatt.distinct.count | ||
| + | |||
| + | var t2= java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | println(" | ||
| + | |||
| + | |||
| + | // ----------------------------------------------------------- | ||
| + | // LUBM 12 : WS | ||
| + | // Pattern: (y worksFor z) (z subOrganisation t) | ||
| + | // ----------------------------------------------------------- | ||
| + | |||
| + | val t1 = java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | val patws = quadsDist.filter({case(i, | ||
| + | |||
| + | val ans_patws = patws.distinct.count | ||
| + | |||
| + | val t2= java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | println(" | ||
| + | </ | ||
| + | |||
| + | ===Warp=== | ||
| < | < | ||
| // Spark implementation of WARP replication | // Spark implementation of WARP replication | ||
| Ligne 289: | Ligne 406: | ||
| val folderName = folder + scale | val folderName = folder + scale | ||
| val fileName = dataset + scale | val fileName = dataset + scale | ||
| + | /** | ||
| + | * set inputData with the path to the data encoded as quadruples (see Datasets excerpts) | ||
| + | */ | ||
| val inputData = s"/ | val inputData = s"/ | ||
| Ligne 616: | Ligne 736: | ||
| </ | </ | ||
| + | ===2-hop based approach=== | ||
| + | < | ||
| + | val folder= | ||
| + | val dataset= " | ||
| + | val scale=" | ||
| + | val folderName = folder +scale | ||
| + | val part = Array(5, | ||
| + | |||
| + | for (p <- part) | ||
| + | { | ||
| + | val fileName = dataset+scale+" | ||
| + | val fileNamewatdiv2k_encoded_unique_quads.partNew.5 | ||
| + | val t1 = java.lang.System.currentTimeMillis(); | ||
| + | |||
| + | val quads = sc.textFile(s"/ | ||
| + | |||
| + | var addOneHop = quads.map({case(s, | ||
| + | |||
| + | val newQuads = quads.union(addOneHop).distinct | ||
| + | val newQuadsSize = newQuads.count | ||
| + | |||
| + | val t2 = java.lang.System.currentTimeMillis(); | ||
| + | val hopSize = addOneHop.count | ||
| + | println(s" | ||
| + | println(s" | ||
| + | newQuads.saveAsTextFile(s"/ | ||
| + | } | ||
| + | </ | ||
| ====Datasets excerpts==== | ====Datasets excerpts==== | ||
| + | ===Encoding of LUBM concepts and properties=== | ||
| + | |||
| + | < | ||
| + | Properties: | ||
| + | 0 < | ||
| + | 603979776 < | ||
| + | 671088640 < | ||
| + | 738197504 < | ||
| + | 805306368 < | ||
| + | 872415232 < | ||
| + | 939524096 < | ||
| + | 1006632960 < | ||
| + | 1082130432 < | ||
| + | 1090519040 < | ||
| + | 1098907648 < | ||
| + | 1101004800 < | ||
| + | 1103101952 < | ||
| + | 1105199104 < | ||
| + | 1107296256 < | ||
| + | 1115684864 < | ||
| + | 1124073472 < | ||
| + | 1132462080 < | ||
| + | 1136656384 < | ||
| + | 1138753536 < | ||
| + | 1140850688 < | ||
| + | 1149239296 < | ||
| + | 1157627904 < | ||
| + | 1166016512 < | ||
| + | 1174405120 < | ||
| + | 1182793728 < | ||
| + | 1191182336 < | ||
| + | 1199570944 < | ||
| + | 1207959552 < | ||
| + | 1216348160 < | ||
| + | 1224736768 < | ||
| + | 1233125376 < | ||
| + | 1241513984 < | ||
| + | |||
| + | Concepts: | ||
| + | 0 < | ||
| + | 268435456 < | ||
| + | 301989888 < | ||
| + | 335544320 < | ||
| + | 369098752 < | ||
| + | 402653184 < | ||
| + | 436207616 < | ||
| + | 469762048 < | ||
| + | 536870912 < | ||
| + | 570425344 < | ||
| + | 603979776 < | ||
| + | 637534208 < | ||
| + | 671088640 < | ||
| + | 704643072 < | ||
| + | 713031680 < | ||
| + | 721420288 < | ||
| + | 729808896 < | ||
| + | 738197504 < | ||
| + | 805306368 < | ||
| + | 872415232 < | ||
| + | 939524096 < | ||
| + | 956301312 < | ||
| + | 973078528 < | ||
| + | 1006632960 < | ||
| + | 1015021568 < | ||
| + | 1023410176 < | ||
| + | 1031798784 < | ||
| + | 1033895936 < | ||
| + | 1035993088 < | ||
| + | 1040187392 < | ||
| + | 1042284544 < | ||
| + | 1044381696 < | ||
| + | 1044643840 < | ||
| + | 1044905984 < | ||
| + | 1045168128 < | ||
| + | 1045430272 < | ||
| + | 1045692416 < | ||
| + | 1045954560 < | ||
| + | 1046478848 < | ||
| + | 1073741824 < | ||
| + | 1140850688 < | ||
| + | 1174405120 < | ||
| + | 1207959552 < | ||
| + | </ | ||
| + | |||
| + | ===LUBM Univ1 === | ||
| + | -[[http:// | ||
| + | |||
| + | -[[http:// | ||
| + | |||
| + | -[[http:// | ||
| + | |||
| + | |||
| + | -[[http:// | ||
site/recherche/logiciels/rdfdist.1431345688.txt.gz · Dernière modification : de amine