diff --git a/Java/MR_DomainIpTest.java b/Java/MR_DomainIpTest.java new file mode 100644 index 0000000000000000000000000000000000000000..a0bdaf94238feb7c7b562f6b54bc1542b49ea915 --- /dev/null +++ b/Java/MR_DomainIpTest.java @@ -0,0 +1,111 @@ +package first; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.MultiFileInputFormat; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import java.net.URI; +import java.io.IOException; +import java.net.URI; +import java.text.SimpleDateFormat; +import java.util.Locale; +import org.apache.hadoop.conf.Configuration; + +public class MR_DomainIpTest { + //读取逗号分隔符map + public static class dhMaper extends Mapper{ + private static final String sep = "|"; + String domain_name = new String(); + String ip = new String(); + String datatime = new String(); + String line=new String(); + String[] fields; + String status=new String(); + String res=new String(); + String reslist=new String(); + String domain_name1="xxx.com"; + String domain_name2="xxx.com"; + String domain_name3="xxx.com"; + String domain_name4="xxx.com"; + String domain_name5="xxx.com"; + String domain_name6="xxx.com"; + String domain_name7="xxx.com"; + String domain_name8="xxx.com"; + String domain_name9="xxx.com"; + String domain_name10="xxx.com"; + String domain_name11="xxx.com"; + String domain_name12="xxx.com"; + String domain_name13="xxx.com"; + String domain_name14="xxx.com"; + StringBuilder keySB = new StringBuilder();// Mapper输出Key Buffer + StringBuilder valueSB = new StringBuilder();// Mapper输出value Buffer + MR_demo02.KeyPair mapKey = new MR_demo02.KeyPair();// Map输出key值 + Text mapValue = new Text();// Map输出value值 + + @Override + protected void map(Object key, Text value, Context context) throws IOException, InterruptedException { + line = value.toString(); + + fields = line.split("\\|", -1);// 分割数据 + if(fields.length<11){ + return; + } + domain_name=fields[2].trim(); + ip=fields[5].trim(); + datatime=fields[8].trim().substring(0,14); + status=fields[10].trim(); + res=fields[7].trim(); + reslist=res.split(";",-1).toString(); + if(domain_name.length()<3 || "null".equals(domain_name) ||"".equals(res) || !"r".equals(status)|| reslist.length()<2){ + return; + } + if(!domain_name.contains(domain_name1) && !domain_name.contains(domain_name2) && !domain_name.contains(domain_name3) && + !domain_name.contains(domain_name4) && !domain_name.contains(domain_name5) && !domain_name.contains(domain_name6) + && !domain_name.contains(domain_name7) && !domain_name.contains(domain_name8) && !domain_name.contains(domain_name9) + && !domain_name.contains(domain_name10) && !domain_name.contains(domain_name11) && !domain_name.contains(domain_name12) + && !domain_name.contains(domain_name13) && !domain_name.contains(domain_name14) ){ + return; + } + keySB.setLength(0); + keySB.append(domain_name).toString(); + + //开始时间 + + valueSB.setLength(0); + valueSB.append(domain_name).append(sep); + valueSB.append(ip).append(sep); + valueSB.append(datatime); + + mapValue.set(valueSB.toString()); + context.write(keySB, mapValue); + + + } + } + //读取空格分隔符map + + //reduce函数 + public static class myReducer extends Reducer{ + private static final String sep = "|"; + String domain_name = new String(); + String ip = new String(); + String datatime = new String(); + String line=new String(); + String[] fields; + public void reduce(Text key, Text values, Context context) + throws IOException, InterruptedException { + context.write(values,null); + } + } + +}