Java代碼統(tǒng)計網(wǎng)站中不同省份用戶的訪問數(shù)
一、需求
針對log日志中給定的信息,統(tǒng)計網(wǎng)站中不同省份用戶的訪問數(shù)
二、編程代碼
package org.apache.hadoop.studyhdfs.mapreduce; import java.io.IOException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Mapper.Context; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.jboss.netty.util.internal.StringUtil; public class ProvinceCountMapReduce extends Configured implements Tool { //1.map /* * <KEYIN,VALUEIN,KEYOUT,VALUEOUT> */ public static class WordCountMapper extends Mapper<LongWritable,Text,IntWritable,IntWritable>{ private IntWritable mapOutputKey =new IntWritable(); private IntWritable mapOutputValue =new IntWritable(1); @Override public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { //get lineValue String lineValue =value.toString(); //split String[] strs =lineValue.split("\t"); //line blank String url=strs[1]; String provinceIdValue =strs[23]; //guolv if(strs.length < 30 || StringUtils.isBlank(provinceIdValue) || StringUtils.isBlank(url)){ return; } int provinceId =Integer.MAX_VALUE; try { provinceId=Integer.valueOf(provinceIdValue); } catch (Exception e) { return; } if(provinceId == Integer.MAX_VALUE){ return; } mapOutputKey.set(provinceId); context.write(mapOutputKey, mapOutputValue); } } //2.reduce public static class WordCountReduce extends Reducer<IntWritable,IntWritable,IntWritable,IntWritable>{ private IntWritable outputValue =new IntWritable(); @Override public void reduce(IntWritable key, Iterable<IntWritable> values,Context context) throws IOException, InterruptedException { //to do int sum = 0; for(IntWritable value:values){ sum +=value.get(); } outputValue.set(sum); context.write(key, outputValue); } } public int run(String[] args) throws Exception{ //1.get Configuration Configuration conf =super.getConf(); //2.create job Job job =Job.getInstance(conf, this.getClass().getSimpleName()); job.setJarByClass(ProvinceCountMapReduce.class); //3.set job //3.1 set input Path inputPath =new Path(args[0]); FileInputFormat.addInputPath(job, inputPath); //3.2 set mapper job.setMapperClass(WordCountMapper.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(IntWritable.class); //3.3 set reduce job.setReducerClass(WordCountReduce.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(IntWritable.class); //3.4 set input Path outputPath =new Path(args[1]); FileOutputFormat.setOutputPath(job, outputPath); //4.submmit boolean isSuccess =job.waitForCompletion(true); return isSuccess?0:1; } public static void main(String[] args) throws Exception { args =new String[]{ "hdfs://Hadoop-senior02.beifeng.com:8020/input/2015082818", "hdfs://Hadoop-senior02.beifeng.com:8020/output15/" }; Configuration conf =new Configuration(); conf.set("mapreduce.map.output.compress", "true"); int status=ToolRunner.run(conf, new ProvinceCountMapReduce() , args); System.exit(status); } }
3、運行結果
1)運行代碼:bin/hdfs dfs -text /output15/par*
2)運行結果:
1 3527
2 1672
3 511
4 325
5 776
6 661
7 95
8 80
9 183
10 93
11 135
12 289
13 264
14 374
15 163
16 419
17 306
18 272
19 226
20 2861
21 124
22 38
23 96
24 100
25 20
26 157
27 49
28 21
29 85
30 42
32 173
以上所述是小編給大家介紹的Java代碼統(tǒng)計網(wǎng)站中不同省份用戶的訪問數(shù)的相關介紹,希望對大家有所幫助,在此小編也非常感謝大家對腳本之家網(wǎng)站的支持!
相關文章
SpringBoot 使用 @Value 注解讀取配置文件給靜態(tài)變量賦值
這篇文章主要介紹了SpringBoot 使用 @Value 注解讀取配置文件給靜態(tài)變量賦值,文中通過示例代碼介紹的非常詳細,對大家的學習或者工作具有一定的參考學習價值,需要的朋友們下面隨著小編來一起學習學習吧2020-11-11Java基礎知識精通循環(huán)結構與break及continue
循環(huán)結構是指在程序中需要反復執(zhí)行某個功能而設置的一種程序結構。它由循環(huán)體中的條件,判斷繼續(xù)執(zhí)行某個功能還是退出循環(huán),選擇結構用于判斷給定的條件,根據(jù)判斷的結果判斷某些條件,根據(jù)判斷的結果來控制程序的流程2022-04-04