Uncategorized

Max temperarue using hadoop mapreducer

MAXTEMP.JAVA

import java.io.File;
import java.io.IOException;
import java.util.Scanner;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class MaxTemp
{

public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, “max temp”);
job.setJarByClass(MaxTemp.class);
job.setMapperClass(MaxTempMapper.class);
job.setCombinerClass(MaxTempReducer.class);
job.setReducerClass(MaxTempReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
// System.exit(job.waitForCompletion(true) ? 0 : 1);
job.waitForCompletion(true);

FileSystem fs= FileSystem.get(conf);

FileStatus status[]=fs.listStatus(new Path(args[1]));

for(int i=0;i<status.length;i++)
{
fs.copyToLocalFile(false, new Path(“/user/hduser/”+args[1]),new Path(“/home/hduser/”+args[1]));
}

Scanner s= new Scanner(new File(“/home/hduser/”+args[1]+”/part-r-00000″));
String year[]= new String[10];
int temp[]=new int[10];
int count=0;
while(s.hasNext())
{
year[count]=s.next();
temp[count]=Integer.parseInt(s.next());
count++;
}
int maxtemp=Integer.MIN_VALUE;
String hottestyear=””;

for(int i=0;i<count;i++)
{
if(temp[i]>maxtemp)
{
maxtemp=temp[i];
hottestyear=year[i];

}
}

System.out.println(“Hottest year:”+hottestyear+”\nTemperature:”+maxtemp);

}
}

MAXTEMPMAPPER.JAVA

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class MaxTempMapper
extends Mapper<Object, Text, Text, IntWritable>{

public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {

String line=value.toString();
String year=line.substring(15,19);
int temp;
if(line.charAt(87)==’+’)
{
temp=Integer.parseInt(line.substring(88,92));
}
else
{
temp=Integer.parseInt(line.substring(87,92));
}

String quality=line.substring(92,93);

if(temp!=9999 && quality.matches(“[01459]”) )
{
context.write(new Text(year), new IntWritable(temp));
}
}
}

MAXTEMPREDUCER.JAVA

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class MaxTempReducer
extends Reducer<Text,IntWritable,Text,IntWritable> {
private IntWritable result = new IntWritable();

public void reduce(Text key, Iterable<IntWritable> values,
Context context
) throws IOException, InterruptedException {
int maxtemp = Integer.MIN_VALUE;
for (IntWritable val : values) {
if(val.get()>maxtemp)
{
maxtemp=val.get();
}
}
result.set(maxtemp);
context.write(key, result);
}

}

Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out /  Change )

Google+ photo

You are commenting using your Google+ account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

Connecting to %s