System processes are now named and supervised

daemon.go

@@ -62,27 +62,29 @@ var (
 
 // Process creates a task and then adds it to processing queue.
 func (d *BaseDaemon) Process(a Actor) {
-	d.enqueue(a, false)
-}
-
-// SystemProcess creates a system task that is restarted in case of failure
-// and then adds it to processing queue.
-func (d *BaseDaemon) SystemProcess(a Actor) {
-	d.enqueue(a, true)
-}
-
-func (d *BaseDaemon) enqueue(a Actor, system bool) {
 	d.queue <- &task{
 		daemon:    d.self,
 		actor:     a,
 		createdAt: time.Now(),
-		system:    system,
+	}
+}
+
+// SystemProcess creates a system task that is restarted in case of failure
+// and then adds it to processing queue.
+func (d *BaseDaemon) SystemProcess(name string, a Actor) {
+	d.queue <- &task{
+		daemon:    d.self,
+		actor:     a,
+		createdAt: time.Now(),
+		system:    true,
+		name:      name,
 	}
 }
 
 // Subscribe subscriasdsdfsdgdfgdfsg sdgsdfg sdfgs dfgdfgdfg.
 func (d *BaseDaemon) Subscribe(topic string, fun interface{}) {
-	d.SystemProcess(func() {
+	name := fmt.Sprintf("%s subscription for topic %q", d.String(), topic)
+	d.SystemProcess(name, func() {
 		if d.subscribeFunc == nil {
 			panic(errMissingSubscriptionFun)
 		}
@@ -151,13 +153,11 @@ func (d *BaseDaemon) base() *BaseDaemon {
 	return d
 }
 
-func (d *BaseDaemon) handlePanic() {
-	if err := recover(); err != nil {
-		d.stats.registerError()
-		if d.panicHandler != nil {
-			d.panicHandler()
-		}
-		log.Printf("Daemon %s recovered from panic. Error: %v\n", d, err)
-		debug.PrintStack()
+func (d *BaseDaemon) handlePanic(err interface{}) {
+	d.stats.registerError()
+	if d.panicHandler != nil {
+		d.panicHandler()
 	}
+	log.Printf("Daemon %s recovered from a panic\nError: %v\n", d, err)
+	debug.PrintStack()
 }

example/daemons/number_printer.go

@@ -19,14 +19,17 @@ func (n *NumberPrinter) Startup() {
 		log.Println("Oh, crap! There was a panic, take a look:")
 	})
 
-	n.SystemProcess(n.enqueue)
+	n.SystemProcess("Random Number Generator", n.generateNumbers)
 }
 
 // Shutdown is empty due to the lack of cleanup.
 func (n *NumberPrinter) Shutdown() {}
 
-func (n *NumberPrinter) enqueue() {
+func (n *NumberPrinter) generateNumbers() {
 	for n.Continue() {
+		if rand.Intn(7) == 0 {
+			panic("Number generator don't work on Sundays!")
+		}
 		// Generate a random number between 1000 and 9999 and print it
 		num := 1000 + rand.Intn(9000)
 		n.Process(n.makeActor(num))
@@ -39,10 +42,10 @@ func (n *NumberPrinter) enqueue() {
 func (n *NumberPrinter) makeActor(num int) satan.Actor {
 	return func() {
 		// Making it crash sometimes
-		if rand.Intn(20) == 0 {
-			panic("Noooooooooo!")
+		if rand.Intn(10) == 0 {
+			panic("Nooooo! Random number generator returned a zero!")
 		}
 
-		log.Println("NumberPrinter says", num)
+		log.Println("Number printer says:", num)
 	}
 }

satan.go

@@ -1,7 +1,9 @@
 package satan
 
 import (
+	"fmt"
 	"log"
+	"runtime/debug"
 	"sync"
 	"time"
 )
@@ -43,6 +45,7 @@ type task struct {
 	actor     Actor
 	createdAt time.Time
 	system    bool
+	name      string
 }
 
 const (
@@ -108,8 +111,6 @@ func (s *Satan) runWorker(i int) {
 		case t := <-s.queue:
 			dur := time.Now().UnixNano() - t.createdAt.UnixNano()
 			s.latency.add(time.Duration(dur))
-
-			// log.Printf("Daemon #%d got some job to do!", i+1)
 			s.processTask(t)
 		default:
 			select {
@@ -122,7 +123,20 @@ func (s *Satan) runWorker(i int) {
 }
 
 func (s *Satan) processTask(t *task) {
-	defer t.daemon.base().handlePanic()
+	defer func() {
+		if err := recover(); err != nil {
+			if t.system {
+				log.Printf("System process %s recovered from a panic\nError: %v\n", t, err)
+				debug.PrintStack()
+
+				// Restarting system task
+				s.queue <- t
+			} else {
+				t.daemon.base().handlePanic(err)
+			}
+		}
+	}()
+
 	start := time.Now()
 
 	t.actor() // <--- THE ACTION HAPPENS HERE
@@ -130,3 +144,11 @@ func (s *Satan) processTask(t *task) {
 	dur := time.Now().UnixNano() - start.UnixNano()
 	t.daemon.base().stats.add(time.Duration(dur))
 }
+
+func (t *task) String() string {
+	if t.name == "" {
+		return fmt.Sprintf("[unnamed %s process]", t.daemon.base())
+	}
+
+	return fmt.Sprintf("%s[%s]", t.daemon.base(), t.name)
+}