我解决了。。两种方法我都实现了。。
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
object WordCount {
def main(args: Array[String]) {
val conf = new SparkConf()
val sc = new SparkContext(conf)
val line = sc.textFile("mylog")
val result1 = line.map(x => x.split("\\|")(24))
val result2 = result1.map(x => x.split(",")(1))
val result3 = result2.map((_,1)).reduceByKey(_+_).sortBy(_._2, false).take(10)
/*value sort other way*/
//val result4 = result2.map((_,1)).reduceByKey(_+_).map(x => (x._2, x._1)).sortByKey(false).take(10)
result3.foreach(println)
sc.stop()
}
} |