wc pass
This commit is contained in:
190
src/mr/worker.go
190
src/mr/worker.go
@@ -10,6 +10,7 @@ import (
|
||||
"net/rpc"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -19,6 +20,15 @@ type KeyValue struct {
|
||||
Value string
|
||||
}
|
||||
|
||||
// Sorting
|
||||
// for sorting by key.
|
||||
type ByKey []KeyValue
|
||||
|
||||
// for sorting by key.
|
||||
func (a ByKey) Len() int { return len(a) }
|
||||
func (a ByKey) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a ByKey) Less(i, j int) bool { return a[i].Key < a[j].Key }
|
||||
|
||||
// use ihash(key) % NReduce to choose the reduce
|
||||
// task number for each KeyValue emitted by Map.
|
||||
func ihash(key string) int {
|
||||
@@ -31,69 +41,109 @@ func ihash(key string) int {
|
||||
func Worker(mapf func(string, string) []KeyValue,
|
||||
reducef func(string, []string) string) {
|
||||
// Your worker implementation here.
|
||||
taskType, inFilename, nReduce := RequestTask()
|
||||
fmt.Printf("Perform %s task on: %s", taskType, inFilename)
|
||||
file, err := os.Open(inFilename)
|
||||
if err != nil {
|
||||
log.Fatalf("cannot open %v", inFilename)
|
||||
}
|
||||
content, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
log.Fatalf("cannot read %v", inFilename)
|
||||
}
|
||||
file.Close()
|
||||
if taskType == "map" {
|
||||
intermediateFiles := []string{}
|
||||
kva := mapf(inFilename, string(content))
|
||||
//hash the map job id
|
||||
mapTaskIdx := ihash(inFilename)
|
||||
for i := range kva {
|
||||
log.Printf("%s: %s", kva[i].Key, kva[i].Value)
|
||||
idx := ihash(kva[i].Key) % nReduce
|
||||
mapOutFn := fmt.Sprintf("mr-%d-%d", mapTaskIdx, idx)
|
||||
// add to intermediate file list
|
||||
intermediateFiles = append(intermediateFiles, mapOutFn)
|
||||
// write intermidiate files
|
||||
data, err := json.MarshalIndent(kva[i], "", " ")
|
||||
taskType := "init"
|
||||
for taskType != "done" {
|
||||
workerId, taskType, inFilenames, nReduce := RequestTask()
|
||||
if taskType == "done" {
|
||||
return
|
||||
}
|
||||
if taskType == "map" {
|
||||
ifn := inFilenames[0]
|
||||
file, err := os.Open(ifn)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to serialize map to JSON: %w", err)
|
||||
log.Fatalf("cannot open %v", ifn)
|
||||
}
|
||||
if err = WriteTempAtomic(mapOutFn, data); err != nil {
|
||||
content, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
log.Fatalf("cannot read %v", ifn)
|
||||
}
|
||||
file.Close()
|
||||
kva := mapf(ifn, string(content))
|
||||
//hash the map job id
|
||||
mapTaskIdx := ihash(ifn)
|
||||
|
||||
IntermediaryFileMaps := make(map[int][]KeyValue)
|
||||
|
||||
for i := 0; i < nReduce; i++ {
|
||||
IntermediaryFileMaps[i] = []KeyValue{}
|
||||
}
|
||||
|
||||
for i := range kva {
|
||||
//log.Printf("%s: %s", kva[i].Key, kva[i].Value)
|
||||
idx := ihash(kva[i].Key) % nReduce
|
||||
IntermediaryFileMaps[idx] = append(IntermediaryFileMaps[idx], kva[i])
|
||||
}
|
||||
|
||||
intermediateFiles := []string{}
|
||||
for idx := range IntermediaryFileMaps {
|
||||
outFile := fmt.Sprintf("mr-%d-%d", mapTaskIdx, idx)
|
||||
intermediateFiles = append(intermediateFiles, outFile)
|
||||
jsonData, err := json.Marshal(IntermediaryFileMaps[idx])
|
||||
if err != nil {
|
||||
log.Fatal("failed to marshal json")
|
||||
}
|
||||
if err = WriteTempAtomic(outFile, jsonData); err != nil {
|
||||
log.Fatalf("failed to write KV map to file")
|
||||
}
|
||||
}
|
||||
SetTaskDone(workerId, "map", intermediateFiles)
|
||||
|
||||
} else if taskType == "reduce" {
|
||||
var intermediate []KeyValue
|
||||
|
||||
for _, f := range inFilenames {
|
||||
// Open the file
|
||||
file, err := os.Open(f)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to open file: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Read the file contents
|
||||
data, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to read file: %v", err)
|
||||
}
|
||||
|
||||
// Parse the JSON data
|
||||
var intermidateShard []KeyValue
|
||||
if err := json.Unmarshal(data, &intermidateShard); err != nil {
|
||||
log.Fatalf("Failed to parse JSON: %v", err)
|
||||
}
|
||||
intermediate = append(intermediate, intermidateShard...)
|
||||
}
|
||||
|
||||
sort.Sort(ByKey(intermediate))
|
||||
|
||||
// Prepare output file
|
||||
reduceJobNum := strings.Split(inFilenames[0], "-")[2]
|
||||
oFilename := fmt.Sprintf("mr-out-%s", reduceJobNum)
|
||||
|
||||
var buffer bytes.Buffer
|
||||
i := 0
|
||||
for i < len(intermediate) {
|
||||
j := i + 1
|
||||
for j < len(intermediate) && intermediate[j].Key == intermediate[i].Key {
|
||||
j++
|
||||
}
|
||||
values := []string{}
|
||||
for k := i; k < j; k++ {
|
||||
values = append(values, intermediate[k].Value)
|
||||
}
|
||||
output := reducef(intermediate[i].Key, values)
|
||||
|
||||
// this is the correct format for each line of Reduce output.
|
||||
fmt.Fprintf(&buffer, "%v %v\n", intermediate[i].Key, output)
|
||||
//fmt.Println(buffer.String())
|
||||
i = j
|
||||
}
|
||||
if err := WriteTempAtomic(oFilename, buffer.Bytes()); err != nil {
|
||||
log.Fatalf("failed to write KV map to file: %w", err)
|
||||
}
|
||||
SetTaskDone(workerId, "reduce", []string{})
|
||||
}
|
||||
SetTaskDone("map", inFilename, intermediateFiles)
|
||||
return
|
||||
|
||||
} else if taskType == "reduce" {
|
||||
intermediate := []KeyValue{}
|
||||
|
||||
// Prepare output file
|
||||
oFilename := strings.Split(inFilename, ",")[1]
|
||||
var buffer bytes.Buffer
|
||||
i := 0
|
||||
for i < len(intermediate) {
|
||||
j := i + 1
|
||||
for j < len(intermediate) && intermediate[j].Key == intermediate[i].Key {
|
||||
j++
|
||||
}
|
||||
values := []string{}
|
||||
for k := i; k < j; k++ {
|
||||
values = append(values, intermediate[k].Value)
|
||||
}
|
||||
output := reducef(intermediate[i].Key, values)
|
||||
|
||||
// this is the correct format for each line of Reduce output.
|
||||
fmt.Fprintf(&buffer, "%v %v\n", intermediate[i].Key, output)
|
||||
|
||||
i = j
|
||||
}
|
||||
if err = WriteTempAtomic(oFilename, buffer.Bytes()); err != nil {
|
||||
log.Fatalf("failed to write KV map to file: %w", err)
|
||||
}
|
||||
SetTaskDone("reduce", inFilename, []string{})
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func WriteTempAtomic(fn string, data []byte) error {
|
||||
@@ -135,36 +185,38 @@ func WriteTempAtomic(fn string, data []byte) error {
|
||||
}
|
||||
|
||||
// RPC call to set the task status to done
|
||||
func SetTaskDone(taskType string, filename string, intermediateFiles []string) {
|
||||
func SetTaskDone(workerId string, taskType string, intermediateFiles []string) {
|
||||
args := RpcArgument{}
|
||||
args.WorkerId = workerId
|
||||
args.Method = "set_task_done"
|
||||
args.TaskType = taskType
|
||||
args.Filename = filename
|
||||
args.IntermediateFiles = intermediateFiles
|
||||
reply := RpcReply{}
|
||||
ok := call("Coordinator.SetTaskDone", &args, &reply)
|
||||
if ok {
|
||||
fmt.Printf("ok")
|
||||
} else {
|
||||
if !ok {
|
||||
fmt.Printf("call failed!\n")
|
||||
}
|
||||
/*
|
||||
if ok {
|
||||
fmt.Printf("ok\n")
|
||||
} else {
|
||||
fmt.Printf("call failed!\n")
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
// RPC call to request task from coordinator
|
||||
func RequestTask() (string, string, int) {
|
||||
func RequestTask() (string, string, []string, int) {
|
||||
// declare an arg and reply
|
||||
args := RpcArgument{}
|
||||
args.Method = "request_task"
|
||||
reply := RpcReply{}
|
||||
|
||||
ok := call("Coordinator.GetTask", &args, &reply)
|
||||
if ok {
|
||||
fmt.Printf("reply.TaskType %v\n", reply.TaskType)
|
||||
fmt.Printf("reply.InFilename %v\n", reply.Filename)
|
||||
} else {
|
||||
ok := call("Coordinator.GetTasks", &args, &reply)
|
||||
if !ok {
|
||||
fmt.Printf("call failed!\n")
|
||||
}
|
||||
return reply.TaskType, reply.Filename, reply.NReduce
|
||||
return reply.WorkerId, reply.TaskType, reply.Filenames, reply.NReduce
|
||||
}
|
||||
|
||||
// send an RPC request to the coordinator, wait for the response.
|
||||
|
||||
Reference in New Issue
Block a user