我们的数据一天就一个T，数据量不断增大，集群磁盘有限，所以把冷数据放到了oss,偶尔会使用到冷数据，如果使用的时候还的从oss上拉数据这样很浪费时间后来想了个办法可以直接获取到oss上的数据。 案例： object OssWc {
    def main(args: Array[String]): Unit = {
      if (args.length < 3) {
        System.err.println(         """           |Arguments:           |    inputPath        Input OSS object path, like oss://accessKeyId:accessKeySecret@bucket.endpoint/input/words.txt           |    outputPath       Output OSS object path, like oss://accessKeyId:accessKeySecret@bucket.endpoint/output           |    numPartitions    The number of RDD partitions.           |         """.stripMargin)       System.exit(1)     }     Logger.getLogger("org").setLevel(Level.WARN)     val conf = new SparkConf().setAppName("OssWc")//.setMaster("local[4]")     conf.set("spark.hadoop.fs.oss.impl", "com.aliyun.fs.oss.nat.NativeOssFileSystem")     conf.set("spark.hadoop.mapreduce.job.run-local", "true")     conf.set("spark.hadoop.fs.oss.accessKeyId", "SLFNEWKBG")     conf.set("spark.hadoop.fs.oss.accessKeySecret", "SDFSFSS")     val sc = new SparkContext(conf)     val inputPath = args(0)     val outputPath = args(1)     val numPartitions = 1     val input = sc.textFile(inputPath,numPartitions)     val output = input.flatMap(_.split(",")).map(x=>(x,1)).reduceByKey(_+_).foreach(println(_))     output.saveAsTextFile(outputPath)     sc.stop()   } } 本地测试时注意下 //.setMaster("local[4]") 不然会报以下错：

2018-01-25 12:04:38 [ main:1 ] - [ ERROR ] org.apache.spark.internal.Logging$class.logError(Logging.scala:91) Error initializing SparkContext.

org.apache.spark.SparkException: A master URL must be set in your configuration

at org.apache.spark.SparkContext.<init>(SparkContext.scala:379)

at text.OssWc$.main(OssWc.scala:32)

at text.OssWc.main(OssWc.scala)

Exception in thread "main" org.apache.spark.SparkException: A master URL must be set in your configuration

at org.apache.spark.SparkContext.<init>(SparkContext.scala:379)

at text.OssWc$.main(OssWc.scala:32)

at text.OssWc.main(OssWc.scala)