shezmu/shezmu.go

package shezmu

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime/debug"
	"sync"
	"syscall"
	"time"

	"github.com/localhots/shezmu/stats"
)

// Shezmu is the master daemon.
type Shezmu struct {
	DaemonStats stats.Publisher
	Logger      Logger
	NumWorkers  int

	daemons      []Daemon
	queue        chan *task
	runtimeStats stats.Manager

	wgWorkers       sync.WaitGroup
	wgSystem        sync.WaitGroup
	shutdownWorkers chan struct{}
	shutdownSystem  chan struct{}
}

// Actor is a function that could be executed by daemon workers.
type Actor func()

// Logger is the interface that implements minimal logging functions.
type Logger interface {
	Printf(format string, v ...interface{})
	Println(v ...interface{})
}

type task struct {
	daemon    Daemon
	actor     Actor
	createdAt time.Time
	system    bool
	name      string
}

const (
	// DefaultNumWorkers is the default number of workers that would process
	// tasks.
	DefaultNumWorkers = 100
)

// Summon creates a new instance of Shezmu.
func Summon() *Shezmu {
	return &Shezmu{
		DaemonStats:     &stats.Void{},
		Logger:          log.New(os.Stdout, "", log.LstdFlags),
		NumWorkers:      DefaultNumWorkers,
		queue:           make(chan *task),
		runtimeStats:    stats.NewBasicStats(),
		shutdownWorkers: make(chan struct{}),
		shutdownSystem:  make(chan struct{}),
	}
}

// AddDaemon adds a new daemon.
func (s *Shezmu) AddDaemon(d Daemon) {
	base := d.base()
	base.self = d
	base.queue = s.queue
	base.logger = s.Logger
	base.shutdown = s.shutdownSystem

	s.daemons = append(s.daemons, d)
}

// ClearDaemons clears the list of added daemons. StopDaemons() function MUST be
// called before calling ClearDaemons().
func (s *Shezmu) ClearDaemons() {
	s.daemons = []Daemon{}
}

// StartDaemons starts all registered daemons.
func (s *Shezmu) StartDaemons() {
	s.Logger.Printf("Starting %d workers", s.NumWorkers)
	for i := 0; i < s.NumWorkers; i++ {
		go s.runWorker()
	}

	s.Logger.Println("Setting up daemons")
	for _, d := range s.daemons {
		s.setupDaemon(d)
	}
}

// StopDaemons stops all running daemons.
func (s *Shezmu) StopDaemons() {
	close(s.shutdownSystem)
	for _, d := range s.daemons {
		d.Shutdown()
	}

	s.wgSystem.Wait()
	close(s.shutdownWorkers)

	s.wgWorkers.Wait()
	close(s.queue)

	// Re-open closed channels to allow starting new deamons afterwards
	s.shutdownSystem = make(chan struct{})
	s.shutdownWorkers = make(chan struct{})
	s.queue = make(chan *task)

	fmt.Println(s.runtimeStats.Fetch(stats.Latency))
}

func (s *Shezmu) HandleSignals() {
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT)
	for {
		switch sig := <-ch; sig {
		case syscall.SIGINT:
			s.StopDaemons()
			return
		default:
			s.Logger.Printf("Signal ignored: %s", sig)
		}
	}
}

func (s *Shezmu) setupDaemon(d Daemon) {
	defer func() {
		if err := recover(); err != nil {
			s.Logger.Printf("Failed to setup daemon %s due to process termination", d)
		}
	}()

	s.queue <- &task{
		daemon:    d,
		actor:     d.Startup,
		createdAt: time.Now(),
		system:    true,
		name:      "startup",
	}
}

func (s *Shezmu) runWorker() {
	s.wgWorkers.Add(1)
	defer s.wgWorkers.Done()
	defer func() {
		if err := recover(); err != nil {
			s.Logger.Printf("Worker crashed. Error: %v\n", err)
			debug.PrintStack()
			go s.runWorker() // Restarting worker
		}
	}()

	for {
		select {
		case t := <-s.queue:
			s.processTask(t)
		case <-s.shutdownWorkers:
			return
		}
	}
}

func (s *Shezmu) processTask(t *task) {
	dur := time.Now().Sub(t.createdAt)
	s.runtimeStats.Add(stats.Latency, dur)

	if t.system {
		s.processSystemTask(t)
	} else {
		s.processGeneralTask(t)
	}
}

func (s *Shezmu) processSystemTask(t *task) {
	// Abort starting a system task if shutdown was already called. Otherwise
	// incrementing a wait group counter will cause a panic. This should be an
	// extremely rare scenario when a system task crashes and tries to restart
	// after a shutdown call.
	select {
	case <-s.shutdownSystem:
		return
	default:
	}

	s.wgSystem.Add(1)
	defer s.wgSystem.Done()
	defer func() {
		if err := recover(); err != nil {
			s.Logger.Printf("System task %s recovered from a panic\nError: %v\n", t, err)
			debug.PrintStack()

			t.createdAt = time.Now()
			s.queue <- t // Restarting task
		} else {
			s.Logger.Printf("System task %s finished\n", t)
		}
	}()

	s.Logger.Printf("Starting system task %s\n", t)
	t.actor() // <--- ACTION STARTS HERE
}

func (s *Shezmu) processGeneralTask(t *task) {
	defer func() {
		if val := recover(); val != nil {
			err := interfaceToError(val)
			s.DaemonStats.Error(t.daemon.String())
			t.daemon.base().handlePanic(err)
			s.Logger.Printf("Daemon %s recovered from a panic\nError: %s\n", t.daemon, err.Error())
			debug.PrintStack()
		}
	}()
	defer func(start time.Time) {
		dur := time.Now().Sub(start)
		s.DaemonStats.Add(t.daemon.String(), dur)
	}(time.Now())

	t.actor() // <--- ACTION STARTS HERE
}

func (t *task) String() string {
	return fmt.Sprintf("%s[%s]", t.daemon, t.name)
}

func interfaceToError(val interface{}) error {
	if terr, ok := val.(error); ok {
		return terr
	}

	return fmt.Errorf("%v", val)
}