任務(wù)7實(shí)訓(xùn)：MapReduce編程

任務(wù)七實(shí)訓(xùn)：MapReduce編程一、實(shí)驗(yàn)?zāi)康?.1掌握MapReduce的編程方法二、實(shí)驗(yàn)內(nèi)容2.1在idea中新建一個(gè)Maven工程2.2在pom文件中添加依賴添加依賴<dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-client</artifactId><version>2.9.2</version></dependency>2.3新建一個(gè)class命名為WordCount，importorg.apache.hadoop.conf.Configuration;

importorg.apache.hadoop.fs.Path;

importorg.apache.hadoop.io.IntWritable;

importorg.apache.hadoop.io.Text;

importorg.apache.hadoop.mapreduce.Job;

importorg.apache.hadoop.mapreduce.Mapper;

importorg.apache.hadoop.mapreduce.Reducer;

importorg.apache.hadoop.mapreduce.lib.input.FileInputFormat;

importorg.apache.hadoop.mapreduce.lib.input.TextInputFormat;

importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

importorg.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

importjava.io.IOException;

importjava.util.StringTokenizer;publicclassWordCount{

publicstaticclassMyMapperextendsMapper<Object,Text,Text,IntWritable>{

privateTextword=newText();

privatefinalstaticIntWritableone=newIntWritable(1);

@Override

protectedvoidmap(Objectkey,Textvalue,Contextcontext)throwsIOException,InterruptedException{

StringTokenizeritr=newStringTokenizer(value.toString());

while(itr.hasMoreTokens()){

word.set(itr.nextToken());

context.write(word,one);

}

}

}

publicstaticclassMyReducerextendsReducer<Text,IntWritable,Text,IntWritable>{

privateIntWritableresult=newIntWritable();

@Override

protectedvoidreduce(Textkey,Iterable<IntWritable>values,Contextcontext)throwsIOException,InterruptedException{

intsum=0;

for(IntWritableval:values){

sum+=val.get();

}

result.set(sum);

context.write(key,result);

}

}

publicstaticvoidmain(String[]args)throwsIOException,InterruptedException,ClassNotFoundException{

Configurationconf=newConfiguration();

Jobjob=Job.getInstance(conf);

job.setJarByClass(WordCount.class);

job.setJobName("WordCount");

job.setInputFormatClass(TextInputFormat.class);

job.setOutputFormatClass(TextOutputFormat.class);

FileInputFormat.addInputPath(job,newPath(args[0]));

FileOutputFormat.setOutputPath(job,newPath(args[1]));

job.setMapperClass(MyMapper.class);

job.setReducerClass(MyReducer.class);

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(IntWritable.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

job.waitForCompletion(true);

}

}2.4.配置hadoop環(huán)境變量Windows本地解壓Hadoop壓縮包，然后像配置JDK環(huán)境變量一樣在系統(tǒng)環(huán)境變量里配置HADOOP_HOME和path：%HADOOP_HOME\bin%環(huán)境變量。注意：hadoop安裝目錄盡量不要包含空格或者中文字符。2.5.下載hadoop對(duì)應(yīng)版本補(bǔ)丁MapReduce程序在Linux、mac系統(tǒng)中可以直接運(yùn)行，但無(wú)法直接在Windows下運(yùn)行，因?yàn)镸apReduce代碼在Windows下運(yùn)行存在bug，需要添加hadoop補(bǔ)丁才可以。Hadoop補(bǔ)丁下載地址：/cdarlint/winutilsHadoop補(bǔ)丁文件為hadoop.dll和winutils.exe。2.6補(bǔ)丁添加hadoop安裝目錄hadoop補(bǔ)丁文件下載之后，解壓，然后將相應(yīng)版本中的hadoop.dll和winutils.exe補(bǔ)丁文件，拷貝到{HADOOP_HOME}/bin目錄下（可能需要重啟IDEA）2.6補(bǔ)丁添加windowsc盤目錄如果1.3步驟操作完之后，還無(wú)法運(yùn)行MapReduce程序，則需要將hadoop.dll和winutils.exe補(bǔ)丁文件，拷貝