wc pass

src/main/test-mr.sh

@@ -113,6 +113,7 @@ fi
 # wait for remaining workers and coordinator to exit.
 wait
 
+exit 0
 #########################################################
 # now indexer
 rm -f mr-*

src/mr/coordinator.go

@@ -1,60 +1,105 @@
 package mr
 
 import (
+	"crypto/rand"
 	"errors"
+	"fmt"
 	"log"
+	"math/big"
 	"net"
 	"net/http"
 	"net/rpc"
 	"os"
+	"strconv"
+	"strings"
+	"sync"
 	"time"
 )
 
+const (
+	JOB_TIMEOUT_S = 10 * time.Second
+	DEBUG         = true
+)
+
 type Coordinator struct {
 	MapJobs    []Job
 	ReduceJobs []Job
 	NReduce    int
+	mu         sync.Mutex
+	mapWg      sync.WaitGroup
 }
 
 type Job struct {
-	Filename string
+	WorkerId  string
-	Status   bool
+	Filenames []string
-	Active   bool
+	Status    bool
-	LastPing time.Time
+	Active    bool
+	LastPing  time.Time
 }
 
 // Your code here -- RPC handlers for the worker to call.
 //
 // the RPC argument and reply types are defined in rpc.go.
 func (c *Coordinator) GetTasks(args *RpcArgument, reply *RpcReply) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
 	if args.Method != "request_task" {
 		return errors.New("argument .Method not valid for this procedure")
 	}
 	// check for any non-completed OR non-active map jobs
-	for _, mj := range c.MapJobs {
+	for i, mj := range c.MapJobs {
 		// assign job that not already done OR not actively being run by other workers
+		// status = false AND active = false
 		if !mj.Status && !mj.Active {
+			wId, err := genWorkerId()
+			if err != nil {
+				fmt.Println(err)
+			}
 			reply.NReduce = c.NReduce
 			reply.TaskType = "map"
-			reply.Filename = mj.Filename
+			reply.Filenames = mj.Filenames
-			// set job status, taken by a worker
+			reply.WorkerId = wId
-			mj.Active = true
+			// set job status, taken by a worke
-			mj.LastPing = time.Now()
+			c.MapJobs[i].WorkerId = wId
+			c.MapJobs[i].Active = true
+			c.MapJobs[i].LastPing = time.Now()
+			if DEBUG {
+				fmt.Printf("[Map Job Assigned] %s | file: %s | status: %t | active: %t |\n",
+					c.MapJobs[i].WorkerId,
+					c.MapJobs[i].Filenames[0],
+					c.MapJobs[i].Status, c.MapJobs[i].Active)
+			}
 			return nil
 		}
 	}
+	c.mapWg.Wait()
 	// check for reduce jobs
-	for _, rj := range c.ReduceJobs {
+	for i, rj := range c.ReduceJobs {
 		if !rj.Status && !rj.Active {
+			wId, err := genWorkerId()
+			if err != nil {
+				fmt.Println(err)
+			}
+			reply.WorkerId = wId
 			reply.NReduce = c.NReduce
 			reply.TaskType = "reduce"
-			reply.Filename = rj.Filename
+			reply.Filenames = rj.Filenames
 			// set job status, taken by a worker
-			rj.Active = true
+			c.ReduceJobs[i].WorkerId = wId
-			rj.LastPing = time.Now()
+			c.ReduceJobs[i].Active = true
+			c.ReduceJobs[i].LastPing = time.Now()
+			if DEBUG {
+				fmt.Printf("[Reduce Job Assigned] %s | status: %t | active: %t |\n",
+					c.ReduceJobs[i].WorkerId, c.ReduceJobs[i].Status, c.ReduceJobs[i].Active)
+			}
 			return nil
 		}
 	}
+
+	reply.TaskType = "done"
+	reply.NReduce = c.NReduce
+	reply.Filenames = []string{}
 	return nil
 }
 
@@ -64,26 +109,35 @@ func (c *Coordinator) SetTaskDone(args *RpcArgument, reply *RpcReply) error {
 		return errors.New("argument .Method not valid for this procedure")
 	}
 	if args.TaskType == "map" {
-		f := args.Filename
+		wId := args.WorkerId
-		for _, mj := range c.MapJobs {
+		for i, _ := range c.MapJobs {
-			if mj.Filename == f {
+			if c.MapJobs[i].WorkerId == wId {
-				mj.Status = true
+				c.MapJobs[i].Status = true
+				c.MapJobs[i].Active = false
+				c.mapWg.Done()
+				if DEBUG {
+					fmt.Printf("[DONE] %s | file: %s | status: %t | active: %t\n",
+						c.MapJobs[i].WorkerId,
+						c.MapJobs[i].Filenames[0],
+						c.MapJobs[i].Status, c.MapJobs[i].Active)
+				}
 			}
 		}
-		// Create reduce job data struct for each intermediary file returned from worker's map job
+		// Check again
 		for _, ifn := range args.IntermediateFiles {
-			reduceJob := Job{}
+			reduceJobIdx := strings.Split(ifn, "-")[2]
-			reduceJob.Active = false
+			idx, err := strconv.Atoi(reduceJobIdx)
-			reduceJob.Filename = ifn
+			if err != nil {
-			reduceJob.LastPing = time.Now()
+				fmt.Printf("Error converting string to integer: %v\n", err)
-			reduceJob.Status = false
+			}
-			c.ReduceJobs = append(c.ReduceJobs, reduceJob)
+			c.ReduceJobs[idx].Filenames = append(c.ReduceJobs[idx].Filenames, ifn)
 		}
+
 	} else if args.TaskType == "reduce" {
-		f := args.Filename
+		wId := args.WorkerId
-		for _, rj := range c.ReduceJobs {
+		for i, _ := range c.ReduceJobs {
-			if rj.Filename == f {
+			if c.ReduceJobs[i].WorkerId == wId {
-				rj.Status = true
+				c.ReduceJobs[i].Status = true
 			}
 		}
 	} else {
@@ -106,6 +160,23 @@ func (c *Coordinator) server() {
 	go http.Serve(l, nil)
 }
 
+// check scheduled tasks; if their age > JOB_TIMEOUT_S
+// reset the job status, making it available for scheduling
+func (c *Coordinator) CheckTaskValidity() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for i, rj := range c.ReduceJobs {
+		timeNow := time.Now()
+		if rj.Active && !rj.Status {
+			duration := timeNow.Sub(rj.LastPing)
+			if duration >= JOB_TIMEOUT_S {
+				c.ReduceJobs[i].Active = false
+				c.ReduceJobs[i].Status = false
+			}
+		}
+	}
+}
+
 // main/mrcoordinator.go calls Done() periodically to find out
 // if the entire job has finished.
 func (c *Coordinator) Done() bool {
@@ -116,6 +187,11 @@ func (c *Coordinator) Done() bool {
 			return ret
 		}
 	}
+	for _, mj := range c.ReduceJobs {
+		if !mj.Status {
+			return ret
+		}
+	}
 	// all mj completed
 	ret = true
 	return ret
@@ -131,12 +207,34 @@ func MakeCoordinator(files []string, nReduce int) *Coordinator {
 
 	for i := range files {
 		mj := Job{}
-		mj.Filename = files[i]
+		mj.Filenames = []string{files[i]}
 		mj.Status = false
 		mj.Active = false
+		c.mapWg.Add(1)
 		c.MapJobs = append(c.MapJobs, mj)
 	}
 
+	for j := 0; j < nReduce; j++ {
+		reduceJob := Job{}
+		reduceJob.Active = false
+		reduceJob.Filenames = []string{}
+		reduceJob.Status = false
+		c.ReduceJobs = append(c.ReduceJobs, reduceJob)
+	}
+
 	c.server()
 	return &c
 }
+
+func genWorkerId() (string, error) {
+	const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+	result := make([]byte, 8)
+	for i := range result {
+		index, err := rand.Int(rand.Reader, big.NewInt(int64(len(charset))))
+		if err != nil {
+			return "", fmt.Errorf("error generating random number: %v", err)
+		}
+		result[i] = charset[index.Int64()]
+	}
+	return string(result), nil
+}

src/mr/rpc.go

@@ -17,16 +17,17 @@ import (
 //
 
 type RpcArgument struct {
+	WorkerId          string
 	Method            string
-	Filename          string
 	TaskType          string
 	IntermediateFiles []string
 }
 
 type RpcReply struct {
-	TaskType string
+	WorkerId  string
-	Filename string
+	TaskType  string
-	NReduce  int
+	Filenames []string
+	NReduce   int
 }
 
 // Add your RPC definitions here.

src/mr/worker.go

@@ -10,6 +10,7 @@ import (
 	"net/rpc"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 )
 
@@ -19,6 +20,15 @@ type KeyValue struct {
 	Value string
 }
 
+// Sorting
+// for sorting by key.
+type ByKey []KeyValue
+
+// for sorting by key.
+func (a ByKey) Len() int           { return len(a) }
+func (a ByKey) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a ByKey) Less(i, j int) bool { return a[i].Key < a[j].Key }
+
 // use ihash(key) % NReduce to choose the reduce
 // task number for each KeyValue emitted by Map.
 func ihash(key string) int {
@@ -31,69 +41,109 @@ func ihash(key string) int {
 func Worker(mapf func(string, string) []KeyValue,
 	reducef func(string, []string) string) {
 	// Your worker implementation here.
-	taskType, inFilename, nReduce := RequestTask()
+	taskType := "init"
-	fmt.Printf("Perform %s task on: %s", taskType, inFilename)
+	for taskType != "done" {
-	file, err := os.Open(inFilename)
+		workerId, taskType, inFilenames, nReduce := RequestTask()
-	if err != nil {
+		if taskType == "done" {
-		log.Fatalf("cannot open %v", inFilename)
+			return
-	}
+		}
-	content, err := io.ReadAll(file)
+		if taskType == "map" {
-	if err != nil {
+			ifn := inFilenames[0]
-		log.Fatalf("cannot read %v", inFilename)
+			file, err := os.Open(ifn)
-	}
-	file.Close()
-	if taskType == "map" {
-		intermediateFiles := []string{}
-		kva := mapf(inFilename, string(content))
-		//hash the map job id
-		mapTaskIdx := ihash(inFilename)
-		for i := range kva {
-			log.Printf("%s: %s", kva[i].Key, kva[i].Value)
-			idx := ihash(kva[i].Key) % nReduce
-			mapOutFn := fmt.Sprintf("mr-%d-%d", mapTaskIdx, idx)
-			// add to intermediate file list
-			intermediateFiles = append(intermediateFiles, mapOutFn)
-			// write intermidiate files
-			data, err := json.MarshalIndent(kva[i], "", " ")
 			if err != nil {
-				log.Fatalf("failed to serialize map to JSON: %w", err)
+				log.Fatalf("cannot open %v", ifn)
 			}
-			if err = WriteTempAtomic(mapOutFn, data); err != nil {
+			content, err := io.ReadAll(file)
+			if err != nil {
+				log.Fatalf("cannot read %v", ifn)
+			}
+			file.Close()
+			kva := mapf(ifn, string(content))
+			//hash the map job id
+			mapTaskIdx := ihash(ifn)
+
+			IntermediaryFileMaps := make(map[int][]KeyValue)
+
+			for i := 0; i < nReduce; i++ {
+				IntermediaryFileMaps[i] = []KeyValue{}
+			}
+
+			for i := range kva {
+				//log.Printf("%s: %s", kva[i].Key, kva[i].Value)
+				idx := ihash(kva[i].Key) % nReduce
+				IntermediaryFileMaps[idx] = append(IntermediaryFileMaps[idx], kva[i])
+			}
+
+			intermediateFiles := []string{}
+			for idx := range IntermediaryFileMaps {
+				outFile := fmt.Sprintf("mr-%d-%d", mapTaskIdx, idx)
+				intermediateFiles = append(intermediateFiles, outFile)
+				jsonData, err := json.Marshal(IntermediaryFileMaps[idx])
+				if err != nil {
+					log.Fatal("failed to marshal json")
+				}
+				if err = WriteTempAtomic(outFile, jsonData); err != nil {
+					log.Fatalf("failed to write KV map to file")
+				}
+			}
+			SetTaskDone(workerId, "map", intermediateFiles)
+
+		} else if taskType == "reduce" {
+			var intermediate []KeyValue
+
+			for _, f := range inFilenames {
+				// Open the file
+				file, err := os.Open(f)
+				if err != nil {
+					log.Fatalf("Failed to open file: %v", err)
+				}
+				defer file.Close()
+
+				// Read the file contents
+				data, err := io.ReadAll(file)
+				if err != nil {
+					log.Fatalf("Failed to read file: %v", err)
+				}
+
+				// Parse the JSON data
+				var intermidateShard []KeyValue
+				if err := json.Unmarshal(data, &intermidateShard); err != nil {
+					log.Fatalf("Failed to parse JSON: %v", err)
+				}
+				intermediate = append(intermediate, intermidateShard...)
+			}
+
+			sort.Sort(ByKey(intermediate))
+
+			// Prepare output file
+			reduceJobNum := strings.Split(inFilenames[0], "-")[2]
+			oFilename := fmt.Sprintf("mr-out-%s", reduceJobNum)
+
+			var buffer bytes.Buffer
+			i := 0
+			for i < len(intermediate) {
+				j := i + 1
+				for j < len(intermediate) && intermediate[j].Key == intermediate[i].Key {
+					j++
+				}
+				values := []string{}
+				for k := i; k < j; k++ {
+					values = append(values, intermediate[k].Value)
+				}
+				output := reducef(intermediate[i].Key, values)
+
+				// this is the correct format for each line of Reduce output.
+				fmt.Fprintf(&buffer, "%v %v\n", intermediate[i].Key, output)
+				//fmt.Println(buffer.String())
+				i = j
+			}
+			if err := WriteTempAtomic(oFilename, buffer.Bytes()); err != nil {
 				log.Fatalf("failed to write KV map to file: %w", err)
 			}
+			SetTaskDone(workerId, "reduce", []string{})
 		}
-		SetTaskDone("map", inFilename, intermediateFiles)
-		return
-
-	} else if taskType == "reduce" {
-		intermediate := []KeyValue{}
-
-		// Prepare output file
-		oFilename := strings.Split(inFilename, ",")[1]
-		var buffer bytes.Buffer
-		i := 0
-		for i < len(intermediate) {
-			j := i + 1
-			for j < len(intermediate) && intermediate[j].Key == intermediate[i].Key {
-				j++
-			}
-			values := []string{}
-			for k := i; k < j; k++ {
-				values = append(values, intermediate[k].Value)
-			}
-			output := reducef(intermediate[i].Key, values)
-
-			// this is the correct format for each line of Reduce output.
-			fmt.Fprintf(&buffer, "%v %v\n", intermediate[i].Key, output)
-
-			i = j
-		}
-		if err = WriteTempAtomic(oFilename, buffer.Bytes()); err != nil {
-			log.Fatalf("failed to write KV map to file: %w", err)
-		}
-		SetTaskDone("reduce", inFilename, []string{})
-		return
 	}
+
 }
 
 func WriteTempAtomic(fn string, data []byte) error {
@@ -135,36 +185,38 @@ func WriteTempAtomic(fn string, data []byte) error {
 }
 
 // RPC call to set the task status to done
-func SetTaskDone(taskType string, filename string, intermediateFiles []string) {
+func SetTaskDone(workerId string, taskType string, intermediateFiles []string) {
 	args := RpcArgument{}
+	args.WorkerId = workerId
 	args.Method = "set_task_done"
 	args.TaskType = taskType
-	args.Filename = filename
 	args.IntermediateFiles = intermediateFiles
 	reply := RpcReply{}
 	ok := call("Coordinator.SetTaskDone", &args, &reply)
-	if ok {
+	if !ok {
-		fmt.Printf("ok")
-	} else {
 		fmt.Printf("call failed!\n")
 	}
+	/*
+		if ok {
+			fmt.Printf("ok\n")
+		} else {
+			fmt.Printf("call failed!\n")
+		}
+	*/
 }
 
 // RPC call to request task from coordinator
-func RequestTask() (string, string, int) {
+func RequestTask() (string, string, []string, int) {
 	// declare an arg and reply
 	args := RpcArgument{}
 	args.Method = "request_task"
 	reply := RpcReply{}
 
-	ok := call("Coordinator.GetTask", &args, &reply)
+	ok := call("Coordinator.GetTasks", &args, &reply)
-	if ok {
+	if !ok {
-		fmt.Printf("reply.TaskType %v\n", reply.TaskType)
-		fmt.Printf("reply.InFilename %v\n", reply.Filename)
-	} else {
 		fmt.Printf("call failed!\n")
 	}
-	return reply.TaskType, reply.Filename, reply.NReduce
+	return reply.WorkerId, reply.TaskType, reply.Filenames, reply.NReduce
 }
 
 // send an RPC request to the coordinator, wait for the response.