游戏BGP高防服务器租用 国内低延迟 稳定服务器租用MapReduce Java代码获取文件名称

  1. 11 月前

    游戏BGP高防服务器租用 国内低延迟 稳定服务器租用MapReduce Java代码获取文件名称
    服务器咨询敏敏QQ:2853898501 / 3007425289 电话微信:18316411879

    准备数据
    链接:https://pan.baidu.com/s/1uspTkN6_o-FIPSbq-Q9YCQ
    提取码:u0mf

    下载SumData.txt

    获取文件名称
    在Map中加入以下代码,即可获取读取的文件名称

    //通过context可以获取这行文本所属的文件名称
    FileSplit inputSplit = (FileSplit)context.getInputSplit();
    String filename= inputSplit.getPath().getName();

    代码
    MoreFileMapper

    package com.czxy.day20191118.demo01;

    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.lib.input.FileSplit;

    import java.io.IOException;

    public class MoreFileMapper extends Mapper<LongWritable, Text,Text,Text> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

    //通过context可以获取这行文本所属的文件名称
    FileSplit inputSplit = (FileSplit)context.getInputSplit();
    String filename= inputSplit.getPath().getName();

    context.write(new Text(filename),value);
    }
    }
    MoreFileReducer

    package com.czxy.day20191118.demo01;

    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;

    import java.io.IOException;

    public class MoreFileReducer extends Reducer<Text,Text,Text,Text> {
    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
    //遍历Value进行输出
    for (Text value : values) {
    context.write(key,value);
    }

    }
    }

    MoreFileDriver

    package com.czxy.day20191118.demo01;

    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.conf.Configured;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
    import org.apache.hadoop.util.Tool;
    import org.apache.hadoop.util.ToolRunner;

    public class MoreFileDriver extends Configured implements Tool {
    @Override
    public int run(String[] args) throws Exception {

    Job job = Job.getInstance(new Configuration(),"MoreFile");

    job.setInputFormatClass(TextInputFormat.class);
    TextInputFormat.addInputPath(job,new Path("E:\\cache\\mapReduceTestCache\\SumData.txt"));

    job.setMapperClass(MoreFileMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    job.setReducerClass(MoreFileReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job,new Path("E:\\cache\\mapReduceResultCache\\20191118\\demo01_02"));

    return job.waitForCompletion(true)?0:1;
    }

    public static void main(String[] args) throws Exception {
    ToolRunner.run(new MoreFileDriver(),args);
    }
    }
    运行结果
    QQ截图20191119101954.png

 

或者 注册后参与话题.