| 1990s | 2000s | 2010s | Speedup | |
|---|---|---|---|---|
| Disk Capacity | 2.1 GB | 200 GB | 3,000 GB | 1,400 x |
| Price | $157/GB | $1.05/GB | $0.05/GB | 3,140 x |
| Transfer Rate | 16.6 Mb/sec | 56.5 Mb/sec | 210 Mb/sec | 12 x |
| DataCenter1 | |||
|---|---|---|---|
| Rack 1 | Rack 2 | Rack 3 | |
| Node 1 | Block 1 | Block 1 | |
| Node 2 | Block 2 | Block 3 | |
| Node 3 | Block 4 | Block 3 | |
| DataCenter2 | |||
|---|---|---|---|
| Rack 1 | Rack 2 | ||
| Node 1 | Block 1 | Block 3 | |
| Node 2 | Block 2 | Block 4 | |
| Node 3 | Block 1 | Block 3 | |
| Node 4 | Block 2 | Block 4 | |
namenode -format format the DFS filesystem
secondarynamenode run the DFS secondary namenode
namenode run the DFS namenode
datanode run a DFS datanode
dfsadmin run a DFS admin client
fsck run a DFS filesystem checking utility
fs run a generic filesystem user client
balancer run a cluster balancing utility
jobtracker run the MapReduce job Tracker node
pipes run a Pipes job
tasktracker run a MapReduce tack Tracker node
job manipulate MapReduce jobs
version print the version
jar <jar> run a jar file
distcp <srcurl> <desturl>
copy file or directories recursively
archive -arhiveName NAME <src>* <dest>
create a hadoop archive
daemonlog get/ser the log level for each daemon
CLASSNAME run the class caleed CLASSNAME
public static class MapClass extends MapReduceBase
implements Mapper<LongWritable, Text, Text, IntWritable>
{
private final static IntWritable one = new IntWritable(1);
private Text word = new Text( );
public void map(LongWritable key, Text value,
OutputCollector output,
Reporter reporter) throws IOException {
String line = value.toString();
StringTokenizer itr = new StringTokenizer(line);
while (itr.hasMoreTokens( )) {
word.set(itr.nextToken( ));
output.collect(word, one);
}
}
}
public static class Reduce extends MapReduceBase
implements Reducer<Text, IntWritable, Text, IntWritable>
{
public void reduce(Text key, Iterator<IntWritable> values,
OutputCollector<Text, IntWritable> output,
Reporter reporter) throws IOException {
int sum = 0;
while (values.hasNext( )) {
sum += values.next( ).get( );
}
output.collect(key, new IntWritable(sum));
}
}