Better handling of moving sentinel servers
This commit is contained in:
parent
d51e77efdd
commit
d7b8e82b5c
124
main.go
124
main.go
|
@ -8,13 +8,16 @@ import (
|
||||||
"log"
|
"log"
|
||||||
"net"
|
"net"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
masterAddr *net.TCPAddr
|
saddr *net.TCPAddr // Address of the sentinel service
|
||||||
raddr *net.TCPAddr
|
slock *sync.Mutex // Guard for above var
|
||||||
saddr *net.TCPAddr
|
|
||||||
|
masterAddr *net.TCPAddr // Address of the redis master
|
||||||
|
mlock *sync.Mutex // Guard for above var
|
||||||
|
|
||||||
localAddr = flag.String("listen", ":9999", "local address")
|
localAddr = flag.String("listen", ":9999", "local address")
|
||||||
sentinelAddr = flag.String("sentinel", ":26379", "remote address")
|
sentinelAddr = flag.String("sentinel", ":26379", "remote address")
|
||||||
|
@ -24,15 +27,19 @@ var (
|
||||||
func main() {
|
func main() {
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
|
slock = &sync.Mutex{}
|
||||||
|
mlock = &sync.Mutex{}
|
||||||
|
|
||||||
laddr, err := net.ResolveTCPAddr("tcp", *localAddr)
|
laddr, err := net.ResolveTCPAddr("tcp", *localAddr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal("Failed to resolve local address: %s", err)
|
log.Fatalf("Failed to resolve local address: %s", err.Error())
|
||||||
}
|
|
||||||
saddr, err = net.ResolveTCPAddr("tcp", *sentinelAddr)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal("Failed to resolve sentinel address: %s", err)
|
|
||||||
}
|
}
|
||||||
|
resolveSentinel(*sentinelAddr)
|
||||||
|
|
||||||
|
// If sentinel's address is set to nil, this goroutine will resolve it and set the var again
|
||||||
|
go sentinelUpdater(*sentinelAddr)
|
||||||
|
|
||||||
|
// Continuously query sentinel for the master address, updating masterAddr when needed
|
||||||
go master()
|
go master()
|
||||||
|
|
||||||
listener, err := net.ListenTCP("tcp", laddr)
|
listener, err := net.ListenTCP("tcp", laddr)
|
||||||
|
@ -51,12 +58,41 @@ func main() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func sentinelUpdater(sentinelAddr string) {
|
||||||
|
// Resolve the address of sentinel when needed
|
||||||
|
for {
|
||||||
|
if saddr == nil {
|
||||||
|
log.Print("Resolving sentinel address")
|
||||||
|
resolveSentinel(sentinelAddr)
|
||||||
|
}
|
||||||
|
time.Sleep(1 * time.Second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveSentinel(sentinelAddr string) {
|
||||||
|
// var err error
|
||||||
|
addr, err := net.ResolveTCPAddr("tcp", sentinelAddr)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Failed to resolve sentinel address: %s", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
slock.Lock()
|
||||||
|
saddr = addr
|
||||||
|
slock.Unlock()
|
||||||
|
// TODO other cases when saddr isn't valid
|
||||||
|
}
|
||||||
|
|
||||||
func master() {
|
func master() {
|
||||||
var err error
|
var err error
|
||||||
|
var tempAddr *net.TCPAddr
|
||||||
for {
|
for {
|
||||||
masterAddr, err = getMasterAddr(saddr, *masterName)
|
tempAddr, err = getMasterAddr()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println(err)
|
log.Printf("Failed to get master addres: %s", err.Error())
|
||||||
|
} else {
|
||||||
|
mlock.Lock()
|
||||||
|
masterAddr = tempAddr
|
||||||
|
mlock.Unlock()
|
||||||
}
|
}
|
||||||
time.Sleep(1 * time.Second)
|
time.Sleep(1 * time.Second)
|
||||||
}
|
}
|
||||||
|
@ -78,42 +114,68 @@ func proxy(local io.ReadWriteCloser, remoteAddr *net.TCPAddr) {
|
||||||
go pipe(remote, local)
|
go pipe(remote, local)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getMasterAddr(sentinelAddress *net.TCPAddr, masterName string) (*net.TCPAddr, error) {
|
// Connect to Sentinel and query it to find the redis master
|
||||||
conn, err := net.DialTCP("tcp", nil, sentinelAddress)
|
func getMasterAddr() (*net.TCPAddr, error) {
|
||||||
|
// Connect to sentinel
|
||||||
|
// If the connection times out, that master is probably gone.
|
||||||
|
// Mark saddr as nil so that the resolver thread will update it later.
|
||||||
|
// Create a local copy of the sentinel address, it can change under our feet
|
||||||
|
slock.Lock()
|
||||||
|
if saddr == nil {
|
||||||
|
defer slock.Unlock()
|
||||||
|
return nil, errors.New("Sentinel address not available")
|
||||||
|
}
|
||||||
|
local_saddr := *saddr
|
||||||
|
slock.Unlock()
|
||||||
|
|
||||||
|
sentConn, err := dialTimeout(&local_saddr, 5*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Connecting to sentinel master timed out/failed: %s\n", err.Error())
|
||||||
|
slock.Lock()
|
||||||
|
saddr = nil
|
||||||
|
slock.Unlock()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer sentConn.Close()
|
||||||
|
|
||||||
|
// We connected to the master, ask for the redis master
|
||||||
|
sentConn.Write([]byte(fmt.Sprintf("sentinel get-master-addr-by-name %s\n", *masterName)))
|
||||||
|
|
||||||
|
b := make([]byte, 256)
|
||||||
|
_, err = sentConn.Read(b)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
defer conn.Close()
|
|
||||||
|
|
||||||
conn.Write([]byte(fmt.Sprintf("sentinel get-master-addr-by-name %s\n", masterName)))
|
|
||||||
|
|
||||||
b := make([]byte, 256)
|
|
||||||
_, err = conn.Read(b)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
parts := strings.Split(string(b), "\r\n")
|
parts := strings.Split(string(b), "\r\n")
|
||||||
|
|
||||||
if len(parts) < 5 {
|
if len(parts) < 5 {
|
||||||
err = errors.New("Couldn't get master address from sentinel")
|
err = errors.New("Couldn't get master address from sentinel")
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
//getting the string address for the master node
|
// Parse the address for the master node
|
||||||
stringaddr := fmt.Sprintf("%s:%s", parts[2], parts[4])
|
stringaddr := fmt.Sprintf("%s:%s", parts[2], parts[4])
|
||||||
addr, err := net.ResolveTCPAddr("tcp", stringaddr)
|
addr, err := net.ResolveTCPAddr("tcp", stringaddr)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
//check that there's actually someone listening on that address
|
// Verify the returned address is actually listening
|
||||||
conn2, err := net.DialTCP("tcp", nil, addr)
|
// TODO is this really needed?
|
||||||
if err == nil {
|
conn2, err := dialTimeout(addr, 5*time.Second)
|
||||||
defer conn2.Close()
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
|
defer conn2.Close()
|
||||||
return addr, err
|
return addr, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Connect to a TCPAddr, failing if a timeout is exceeded or other error encountered
|
||||||
|
func dialTimeout(destAddr *net.TCPAddr, timeout time.Duration) (*net.TCPConn, error) {
|
||||||
|
d := net.Dialer{Timeout: timeout}
|
||||||
|
netcon, err := d.Dial("tcp", fmt.Sprintf("%s:%d", destAddr.IP, destAddr.Port))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
conn, _ := netcon.(*net.TCPConn)
|
||||||
|
return conn, nil
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue