@@ -0,0 +1,161 @@ | |||
package main | |||
import ( | |||
"flag" | |||
"fmt" | |||
"log" | |||
"net" | |||
"os" | |||
"strings" | |||
lru "github.com/hashicorp/golang-lru" | |||
"gopkg.in/opensmtpd.v0" | |||
) | |||
var ( | |||
prog = os.Args[0] | |||
skip = []*net.IPNet{} | |||
rbls = []string{ | |||
"b.barracudacentral.org", | |||
"bl.spamcop.net", | |||
"virbl.bit.nl", | |||
"xbl.spamhaus.org", | |||
} | |||
debug bool | |||
masq bool | |||
cache *lru.Cache | |||
) | |||
func debugf(fmt string, args ...interface{}) { | |||
if !debug { | |||
return | |||
} | |||
log.Printf("debug: "+fmt, args...) | |||
} | |||
func reverse(ip net.IP) string { | |||
if ip.To4() == nil { | |||
return "" | |||
} | |||
splitAddress := strings.Split(ip.String(), ".") | |||
for i, j := 0, len(splitAddress)-1; i < len(splitAddress)/2; i, j = i+1, j-1 { | |||
splitAddress[i], splitAddress[j] = splitAddress[j], splitAddress[i] | |||
} | |||
return strings.Join(splitAddress, ".") | |||
} | |||
func lookup(rbl string, host string) (result string, listed bool, err error) { | |||
host = fmt.Sprintf("%s.%s", host, rbl) | |||
var res []string | |||
res, err = net.LookupHost(host) | |||
if listed = len(res) > 0; listed { | |||
txt, _ := net.LookupTXT(host) | |||
if len(txt) > 0 { | |||
result = txt[0] | |||
} | |||
} | |||
// Expected error | |||
if err != nil && strings.HasSuffix(err.Error(), ": no such host") { | |||
err = nil | |||
} | |||
return | |||
} | |||
func onConnect(s *opensmtpd.Session, query *opensmtpd.ConnectQuery) error { | |||
ip := query.Remote.(opensmtpd.Sockaddr).IP() | |||
if ip == nil { | |||
return nil | |||
} | |||
debugf("%s: connect from %s\n", prog, ip) | |||
for _, ipnet := range skip { | |||
if ipnet.Contains(ip) { | |||
debugf("%s: skip %s, IP ignored", prog, ip) | |||
return s.Accept() | |||
} | |||
} | |||
var ( | |||
result string | |||
lsited bool | |||
host = reverse(ip) | |||
err error | |||
) | |||
for _, rbl := range rbls { | |||
if result, listed, err = lookup(rbl, host); err != nil { | |||
log.Printf("%s: %s failed %s: %v\n", prog, rbl, ip, err) | |||
} else if listed { | |||
log.Printf("%s: %s listed %s: %v\n", prog, rbl, ip, result) | |||
cache.Add(s.ID, result) | |||
break | |||
} | |||
} | |||
debugf("%s: pass: %s\n", prog, ip) | |||
if !listed { | |||
// Add negative hit | |||
cache.Add(s.ID, "") | |||
} | |||
return s.Accept() | |||
} | |||
func onDATA(s *opensmtpd.Session) error { | |||
debugf("%s: %s DATA\n", prog, s) | |||
if result, block := cache.Get(s.ID); block && result.(string) != "" { | |||
return s.RejectCode(opensmtpd.FilterClose, 421, result.(string)) | |||
} | |||
return s.Accept() | |||
} | |||
func main() { | |||
cacheSize := flag.Int("cache-size", 1024, "LRU cache size") | |||
rblServer := flag.String("servers", strings.Join(rbls, ","), "RBL servers") | |||
ignoreIPs := flag.String("ignore", "127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,fe80::/64", "ignore IPs") | |||
debugging := flag.Bool("d", false, "be verbose") | |||
verbosity := flag.Bool("v", false, "be verbose") | |||
flag.BoolVar(&masq, "masq", true, "masquerade SMTP banner") | |||
flag.Parse() | |||
debug = *debugging || *verbosity | |||
var err error | |||
if cache, err = lru.New(*cacheSize); err != nil { | |||
log.Fatalln(err) | |||
} | |||
rbls = strings.Split(*rblServer, ",") | |||
for _, prefix := range strings.Split(*ignoreIPs, ",") { | |||
var ipnet *net.IPNet | |||
if _, ipnet, err = net.ParseCIDR(prefix); err != nil { | |||
log.Fatalln(err) | |||
} | |||
skip = append(skip, ipnet) | |||
debugf("ignore: %s\n", ipnet) | |||
} | |||
f := new(opensmtpd.Filter) | |||
f.OnConnect(onConnect) | |||
f.OnDATA(onDATA) | |||
if err = f.Register(); err != nil { | |||
log.Fatalln(err) | |||
} | |||
if err = f.Serve(); err != nil { | |||
log.Fatalln(err) | |||
} | |||
} |
@@ -0,0 +1,11 @@ | |||
package opensmtpd | |||
import ( | |||
"net" | |||
"os" | |||
) | |||
func NewConn(fd int) (net.Conn, error) { | |||
f := os.NewFile(uintptr(fd), "") | |||
return net.FileConn(f) | |||
} |
@@ -0,0 +1,488 @@ | |||
package opensmtpd | |||
import ( | |||
"fmt" | |||
"log" | |||
"net" | |||
"os" | |||
"strings" | |||
lru "github.com/hashicorp/golang-lru" | |||
) | |||
const ( | |||
FilterVersion = 51 | |||
) | |||
const ( | |||
TypeFilterRegister uint32 = iota | |||
TypeFilterEvent | |||
TypeFilterQuery | |||
TypeFilterPipe | |||
TypeFilterResponse | |||
) | |||
var filterTypeName = map[uint32]string{ | |||
TypeFilterRegister: "IMSG_FILTER_REGISTER", | |||
TypeFilterEvent: "IMSG_FILTER_EVENT", | |||
TypeFilterQuery: "IMSG_FILTER_QUERY", | |||
TypeFilterPipe: "IMSG_FILTER_PIPE", | |||
TypeFilterResponse: "IMSG_FILTER_RESPONSE", | |||
} | |||
func filterName(t uint32) string { | |||
if s, ok := filterTypeName[t]; ok { | |||
return s | |||
} | |||
return fmt.Sprintf("UNKNOWN %d", t) | |||
} | |||
const ( | |||
HookConnect = 1 << iota | |||
HookHELO | |||
HookMAIL | |||
HookRCPT | |||
HookDATA | |||
HookEOM | |||
HookReset | |||
HookDisconnect | |||
HookCommit | |||
HookRollback | |||
HookDataLine | |||
) | |||
var hookTypeName = map[uint16]string{ | |||
HookConnect: "HOOK_CONNECT", | |||
HookHELO: "HOOK_HELO", | |||
HookMAIL: "HOOK_MAIL", | |||
HookRCPT: "HOOK_RCPT", | |||
HookDATA: "HOOK_DATA", | |||
HookEOM: "HOOK_EOM", | |||
HookReset: "HOOK_RESET", | |||
HookDisconnect: "HOOK_DISCONNECT", | |||
HookCommit: "HOOK_COMMIT", | |||
HookRollback: "HOOK_ROLLBACK", | |||
HookDataLine: "HOOK_DATALINE", | |||
} | |||
func hookName(h uint16) string { | |||
var s []string | |||
for i := uint(0); i < 11; i++ { | |||
if h&(1<<i) != 0 { | |||
s = append(s, hookTypeName[(1<<i)]) | |||
} | |||
} | |||
return strings.Join(s, ",") | |||
} | |||
const ( | |||
EventConnect = iota | |||
EventReset | |||
EventDisconnect | |||
EventTXBegin | |||
EventTXCommit | |||
EventTXRollback | |||
) | |||
var eventTypeName = map[int]string{ | |||
EventConnect: "EVENT_CONNECT", | |||
EventReset: "EVENT_RESET", | |||
EventDisconnect: "EVENT_DISCONNECT", | |||
EventTXBegin: "EVENT_TX_BEGIN", | |||
EventTXCommit: "EVENT_TX_COMMIT", | |||
EventTXRollback: "EVENT_TX_ROLLBACK", | |||
} | |||
func eventName(t int) string { | |||
if s, ok := eventTypeName[t]; ok { | |||
return s | |||
} | |||
return fmt.Sprintf("UNKNOWN %d", int(t)) | |||
} | |||
const ( | |||
QueryConnect = iota | |||
QueryHELO | |||
QueryMAIL | |||
QueryRCPT | |||
QueryDATA | |||
QueryEOM | |||
QueryDataLine | |||
) | |||
var queryTypeName = map[int]string{ | |||
QueryConnect: "QUERY_CONNECT", | |||
QueryHELO: "QUERY_HELO", | |||
QueryMAIL: "QUERY_MAIL", | |||
QueryRCPT: "QUERY_RCPT", | |||
QueryDATA: "QUERY_DATA", | |||
QueryEOM: "QUERY_EOM", | |||
QueryDataLine: "QUERY_DATALINE", | |||
} | |||
func queryName(t int) string { | |||
if s, ok := queryTypeName[t]; ok { | |||
return s | |||
} | |||
return fmt.Sprintf("UNKNOWN %d", int(t)) | |||
} | |||
const ( | |||
FilterOK = iota | |||
FilterFail | |||
FilterClose | |||
) | |||
var responseTypeName = map[int]string{ | |||
FilterOK: "FILTER_OK", | |||
FilterFail: "FILTER_FAIL", | |||
FilterClose: "FILTER_CLOSE", | |||
} | |||
func responseName(c int) string { | |||
if s, ok := responseTypeName[c]; ok { | |||
return s | |||
} | |||
return fmt.Sprintf("UNKNOWN %d", c) | |||
} | |||
type Filter struct { | |||
Name string | |||
Version uint32 | |||
c net.Conn | |||
m *Message | |||
hooks int | |||
flags int | |||
ready bool | |||
hook struct { | |||
connect func(*Session, *ConnectQuery) error | |||
helo func(*Session, string) error | |||
mail func(*Session, string, string) error | |||
rcpt func(*Session, string, string) error | |||
data func(*Session) error | |||
dataline func(*Session, string) error | |||
eom func(*Session, uint32) error | |||
reset func(*Session) error | |||
disconnect func(*Session) error | |||
commit func(*Session) error | |||
} | |||
session *lru.Cache | |||
} | |||
func (f *Filter) OnConnect(fn func(*Session, *ConnectQuery) error) { | |||
f.hook.connect = fn | |||
f.hooks |= HookConnect | |||
} | |||
func (f *Filter) OnHELO(fn func(*Session, string) error) { | |||
f.hook.helo = fn | |||
f.hooks |= HookHELO | |||
} | |||
func (f *Filter) OnMAIL(fn func(*Session, string, string) error) { | |||
f.hook.mail = fn | |||
f.hooks |= HookMAIL | |||
} | |||
func (f *Filter) OnRCPT(fn func(*Session, string, string) error) { | |||
f.hook.rcpt = fn | |||
f.hooks |= HookRCPT | |||
} | |||
func (f *Filter) OnDATA(fn func(*Session) error) { | |||
f.hook.data = fn | |||
f.hooks |= HookDATA | |||
} | |||
func (f *Filter) OnDataLine(fn func(*Session, string) error) { | |||
f.hook.dataline = fn | |||
f.hooks |= HookDataLine | |||
} | |||
func (f *Filter) Register() error { | |||
var err error | |||
if f.m == nil { | |||
f.m = new(Message) | |||
} | |||
if f.c == nil { | |||
if f.c, err = NewConn(0); err != nil { | |||
return err | |||
} | |||
} | |||
if err = f.m.ReadFrom(f.c); err != nil { | |||
return err | |||
} | |||
if t, ok := filterTypeName[f.m.Type]; ok { | |||
log.Printf("filter: imsg %s\n", t) | |||
} else { | |||
log.Printf("filter: imsg UNKNOWN %d\n", f.m.Type) | |||
} | |||
switch f.m.Type { | |||
case TypeFilterRegister: | |||
var err error | |||
if f.Version, err = f.m.GetTypeUint32(); err != nil { | |||
return err | |||
} | |||
if f.Name, err = f.m.GetTypeString(); err != nil { | |||
return err | |||
} | |||
log.Printf("register version=%d,name=%q\n", f.Version, f.Name) | |||
f.m.reset() | |||
f.m.Type = TypeFilterRegister | |||
f.m.PutTypeInt(f.hooks) | |||
f.m.PutTypeInt(f.flags) | |||
if err = f.m.SendTo(0); err != nil { | |||
return err | |||
} | |||
default: | |||
return fmt.Errorf("filter: unexpected imsg type=%s\n", filterTypeName[f.m.Type]) | |||
} | |||
return nil | |||
} | |||
func (f *Filter) Serve() error { | |||
var err error | |||
if f.m == nil { | |||
f.m = new(Message) | |||
} | |||
if f.session == nil { | |||
if f.session, err = lru.New(1024); err != nil { | |||
return err | |||
} | |||
} | |||
if f.c == nil { | |||
if f.c, err = NewConn(0); err != nil { | |||
return err | |||
} | |||
} | |||
for { | |||
//log.Printf("fdcount: %d [pid=%d]\n", fdCount(), os.Getpid()) | |||
if err := f.m.ReadFrom(f.c); err != nil { | |||
if err.Error() != "resource temporarily unavailable" { | |||
return err | |||
} | |||
} | |||
if err := f.handle(); err != nil { | |||
return err | |||
} | |||
} | |||
} | |||
func (f *Filter) handle() (err error) { | |||
if t, ok := filterTypeName[f.m.Type]; ok { | |||
log.Printf("filter: imsg %s\n", t) | |||
} else { | |||
log.Printf("filter: imsg UNKNOWN %d\n", f.m.Type) | |||
} | |||
switch f.m.Type { | |||
case TypeFilterEvent: | |||
if err = f.handleEvent(); err != nil { | |||
return | |||
} | |||
case TypeFilterQuery: | |||
if err = f.handleQuery(); err != nil { | |||
return | |||
} | |||
} | |||
return | |||
} | |||
func fdCount() int { | |||
d, err := os.Open("/proc/self/fd") | |||
if err != nil { | |||
log.Printf("fdcount open: %v\n", err) | |||
return -1 | |||
} | |||
defer d.Close() | |||
fds, err := d.Readdirnames(-1) | |||
if err != nil { | |||
log.Printf("fdcount: %v\n", err) | |||
return -1 | |||
} | |||
return len(fds) - 1 // -1 for os.Open... | |||
} | |||
func (f *Filter) handleEvent() (err error) { | |||
var ( | |||
id uint64 | |||
t int | |||
) | |||
if id, err = f.m.GetTypeID(); err != nil { | |||
return | |||
} | |||
if t, err = f.m.GetTypeInt(); err != nil { | |||
return | |||
} | |||
log.Printf("imsg event: %s [id=%#x]\n", eventName(t), id) | |||
log.Printf("imsg event data: %q\n", f.m.Data[14:]) | |||
log.Printf("fdcount: %d [pid=%d]\n", fdCount(), os.Getpid()) | |||
switch t { | |||
case EventConnect: | |||
f.session.Add(id, NewSession(f, id)) | |||
case EventDisconnect: | |||
f.session.Remove(id) | |||
} | |||
return | |||
} | |||
func (f *Filter) handleQuery() (err error) { | |||
var ( | |||
id, qid uint64 | |||
t int | |||
) | |||
if id, err = f.m.GetTypeID(); err != nil { | |||
return | |||
} | |||
if qid, err = f.m.GetTypeID(); err != nil { | |||
return | |||
} | |||
if t, err = f.m.GetTypeInt(); err != nil { | |||
return | |||
} | |||
log.Printf("imsg query: %s [id=%#x,qid=%#x]\n", queryName(t), id, qid) | |||
//log.Printf("imsg query data (%d remaining): %q\n", len(f.m.Data[f.m.rpos:]), f.m.Data[f.m.rpos:]) | |||
//log.Printf("fdcount: %d [pid=%d]\n", fdCount(), os.Getpid()) | |||
var s *Session | |||
if cached, ok := f.session.Get(id); ok { | |||
s = cached.(*Session) | |||
} else { | |||
s = NewSession(f, id) | |||
f.session.Add(id, s) | |||
} | |||
s.qtype = t | |||
s.qid = qid | |||
switch t { | |||
case QueryConnect: | |||
var query ConnectQuery | |||
if query.Local, err = f.m.GetTypeSockaddr(); err != nil { | |||
return | |||
} | |||
if query.Remote, err = f.m.GetTypeSockaddr(); err != nil { | |||
return | |||
} | |||
if query.Hostname, err = f.m.GetTypeString(); err != nil { | |||
return | |||
} | |||
log.Printf("query connect: %s\n", query) | |||
if f.hook.connect != nil { | |||
return f.hook.connect(s, &query) | |||
} | |||
log.Printf("filter: WARNING: no connect callback\n") | |||
case QueryHELO: | |||
var line string | |||
if line, err = f.m.GetTypeString(); err != nil { | |||
return | |||
} | |||
log.Printf("query HELO: %q\n", line) | |||
if f.hook.helo != nil { | |||
return f.hook.helo(s, line) | |||
} | |||
log.Printf("filter: WARNING: no HELO callback\n") | |||
return f.respond(s, FilterOK, 0, "") | |||
case QueryMAIL: | |||
var user, domain string | |||
if user, domain, err = f.m.GetTypeMailaddr(); err != nil { | |||
return | |||
} | |||
log.Printf("query MAIL: %s\n", user+"@"+domain) | |||
if f.hook.mail != nil { | |||
return f.hook.mail(s, user, domain) | |||
} | |||
log.Printf("filter: WARNING: no MAIL callback\n") | |||
return f.respond(s, FilterOK, 0, "") | |||
case QueryRCPT: | |||
var user, domain string | |||
if user, domain, err = f.m.GetTypeMailaddr(); err != nil { | |||
return | |||
} | |||
log.Printf("query RCPT: %s\n", user+"@"+domain) | |||
if f.hook.rcpt != nil { | |||
return f.hook.rcpt(s, user, domain) | |||
} | |||
log.Printf("filter: WARNING: no RCPT callback\n") | |||
return f.respond(s, FilterOK, 0, "") | |||
case QueryDATA: | |||
if f.hook.data != nil { | |||
return f.hook.data(s) | |||
} | |||
log.Printf("filter: WARNING: no DATA callback\n") | |||
return f.respond(s, FilterOK, 0, "") | |||
case QueryEOM: | |||
var dataLen uint32 | |||
if dataLen, err = f.m.GetTypeUint32(); err != nil { | |||
return | |||
} | |||
if f.hook.eom != nil { | |||
return f.hook.eom(s, dataLen) | |||
} | |||
log.Printf("filter: WARNING: no EOM callback\n") | |||
return f.respond(s, FilterOK, 0, "") | |||
} | |||
return | |||
} | |||
func (f *Filter) respond(s *Session, status, code int, line string) error { | |||
log.Printf("filter: %s %s [code=%d,line=%q]\n", filterName(TypeFilterResponse), responseName(status), code, line) | |||
if s.qtype == QueryEOM { | |||
// Not implemented | |||
return nil | |||
} | |||
m := new(Message) | |||
m.Type = TypeFilterResponse | |||
m.PutTypeID(s.qid) | |||
m.PutTypeInt(s.qtype) | |||
if s.qtype == QueryEOM { | |||
// Not imlemented | |||
return nil | |||
} | |||
m.PutTypeInt(status) | |||
m.PutTypeInt(code) | |||
if line != "" { | |||
m.PutTypeString(line) | |||
} | |||
if err := m.WriteTo(f.c); err != nil { | |||
log.Printf("filter: respond failed: %v\n", err) | |||
return err | |||
} | |||
return nil | |||
} |
@@ -0,0 +1,315 @@ | |||
package opensmtpd | |||
import ( | |||
"bytes" | |||
"encoding/binary" | |||
"errors" | |||
"fmt" | |||
"io" | |||
"net" | |||
"os" | |||
"syscall" | |||
) | |||
const ( | |||
ibufReadSize = 65535 | |||
imsgMaxSize = 16384 | |||
imsgHeaderSize = 4 + 2 + 2 + 4 + 4 | |||
imsgVersion = 14 | |||
maxLocalPartSize = (255 + 1) | |||
maxDomainPartSize = (255 + 1) | |||
) | |||
type MessageHeader struct { | |||
Type uint32 | |||
Len uint16 | |||
Flags uint16 | |||
PeerID uint32 | |||
PID uint32 | |||
} | |||
type Message struct { | |||
MessageHeader | |||
Data []byte | |||
// rpos is the read position in the current Data | |||
rpos int | |||
// buf is what we read from the socket (and remains) | |||
buf []byte | |||
} | |||
func (m *Message) reset() { | |||
m.Type = 0 | |||
m.Len = 0 | |||
m.Flags = 0 | |||
m.PeerID = imsgVersion | |||
m.PID = uint32(os.Getpid()) | |||
m.Data = m.Data[:0] | |||
m.rpos = 0 | |||
m.buf = m.buf[:0] | |||
} | |||
func (m *Message) ReadFrom(c net.Conn) error { | |||
m.reset() | |||
head := make([]byte, imsgHeaderSize) | |||
if _, err := c.Read(head); err != nil { | |||
return err | |||
} | |||
r := bytes.NewBuffer(head) | |||
if err := binary.Read(r, binary.LittleEndian, &m.MessageHeader); err != nil { | |||
return err | |||
} | |||
data := make([]byte, m.MessageHeader.Len-imsgHeaderSize) | |||
if _, err := c.Read(data); err != nil { | |||
return err | |||
} | |||
m.Data = data | |||
return nil | |||
} | |||
func (m *Message) SendTo(fd int) error { | |||
m.Len = uint16(len(m.Data)) + imsgHeaderSize | |||
buf := new(bytes.Buffer) | |||
//log.Printf("imsg header: %+v\n", m.MessageHeader) | |||
if err := binary.Write(buf, binary.LittleEndian, &m.MessageHeader); err != nil { | |||
return err | |||
} | |||
buf.Write(m.Data) | |||
//log.Printf("imsg send: %d / %q\n", buf.Len(), buf.Bytes()) | |||
return syscall.Sendmsg(fd, buf.Bytes(), nil, nil, 0) | |||
} | |||
func (m *Message) WriteTo(c net.Conn) error { | |||
m.Len = uint16(len(m.Data)) + imsgHeaderSize | |||
buf := new(bytes.Buffer) | |||
//log.Printf("imsg header: %+v\n", m.MessageHeader) | |||
if err := binary.Write(buf, binary.LittleEndian, &m.MessageHeader); err != nil { | |||
return err | |||
} | |||
buf.Write(m.Data) | |||
//log.Printf("imsg send: %d / %q\n", buf.Len(), buf.Bytes()) | |||
_, err := c.Write(buf.Bytes()) | |||
return err | |||
} | |||
func (m *Message) GetInt() (int, error) { | |||
if m.rpos+4 > len(m.Data) { | |||
return 0, io.ErrShortBuffer | |||
} | |||
i := binary.LittleEndian.Uint32(m.Data[m.rpos:]) | |||
m.rpos += 4 | |||
return int(i), nil | |||
} | |||
func (m *Message) GetUint32() (uint32, error) { | |||
if m.rpos+4 > len(m.Data) { | |||
return 0, io.ErrShortBuffer | |||
} | |||
u := binary.LittleEndian.Uint32(m.Data[m.rpos:]) | |||
m.rpos += 4 | |||
return u, nil | |||
} | |||
func (m *Message) GetSize() (uint64, error) { | |||
if m.rpos+8 > len(m.Data) { | |||
return 0, io.ErrShortBuffer | |||
} | |||
u := binary.LittleEndian.Uint64(m.Data[m.rpos:]) | |||
m.rpos += 8 | |||
return u, nil | |||
} | |||
func (m *Message) GetString() (string, error) { | |||
o := bytes.IndexByte(m.Data[m.rpos:], 0) | |||
if o < 0 { | |||
return "", errors.New("imsg: string not NULL-terminated") | |||
} | |||
s := string(m.Data[m.rpos : m.rpos+o]) | |||
m.rpos += o | |||
return s, nil | |||
} | |||
func (m *Message) GetID() (uint64, error) { | |||
if m.rpos+8 > len(m.Data) { | |||
return 0, io.ErrShortBuffer | |||
} | |||
u := binary.LittleEndian.Uint64(m.Data[m.rpos:]) | |||
m.rpos += 8 | |||
return u, nil | |||
} | |||
type Sockaddr []byte | |||
func (sa Sockaddr) IP() net.IP { | |||
switch len(sa) { | |||
case 16: // IPv4, sockaddr_in | |||
return net.IP(sa[4:8]) | |||
case 28: // IPv6, sockaddr_in6 | |||
return net.IP(sa[8:24]) | |||
default: | |||
return nil | |||
} | |||
} | |||
func (sa Sockaddr) Port() uint16 { | |||
switch len(sa) { | |||
case 16: // IPv4, sockaddr_in | |||
return binary.LittleEndian.Uint16(sa[2:4]) | |||
case 28: // IPv6, sockaddr_in6 | |||
return binary.LittleEndian.Uint16(sa[2:4]) | |||
default: | |||
return 0 | |||
} | |||
} | |||
func (sa Sockaddr) Network() string { | |||
return "bla" | |||
} | |||
func (sa Sockaddr) String() string { | |||
return fmt.Sprintf("%s:%d", sa.IP(), sa.Port()) | |||
} | |||
func (m *Message) GetSockaddr() (net.Addr, error) { | |||
s, err := m.GetSize() | |||
if err != nil { | |||
return nil, err | |||
} | |||
if m.rpos+int(s) > len(m.Data) { | |||
return nil, io.ErrShortBuffer | |||
} | |||
a := make(Sockaddr, s) | |||
copy(a[:], m.Data[m.rpos:]) | |||
m.rpos += int(s) | |||
return a, nil | |||
} | |||
func (m *Message) GetMailaddr() (user, domain string, err error) { | |||
var buf [maxLocalPartSize + maxDomainPartSize]byte | |||
if maxLocalPartSize+maxDomainPartSize > len(m.Data[m.rpos:]) { | |||
return "", "", io.ErrShortBuffer | |||
} | |||
copy(buf[:], m.Data[m.rpos:]) | |||
m.rpos += maxLocalPartSize + maxDomainPartSize | |||
user = string(buf[:maxLocalPartSize]) | |||
domain = string(buf[maxLocalPartSize:]) | |||
return | |||
} | |||
func (m *Message) GetType(t uint8) error { | |||
if m.rpos >= len(m.Data) { | |||
return io.ErrShortBuffer | |||
} | |||
b := m.Data[m.rpos] | |||
m.rpos++ | |||
if b != t { | |||
return MProcTypeErr{t, b} | |||
} | |||
return nil | |||
} | |||
func (m *Message) GetTypeInt() (int, error) { | |||
if err := m.GetType(M_INT); err != nil { | |||
return 0, err | |||
} | |||
return m.GetInt() | |||
} | |||
func (m *Message) GetTypeUint32() (uint32, error) { | |||
if err := m.GetType(M_UINT32); err != nil { | |||
return 0, err | |||
} | |||
return m.GetUint32() | |||
} | |||
func (m *Message) GetTypeString() (string, error) { | |||
if err := m.GetType(M_STRING); err != nil { | |||
return "", err | |||
} | |||
return m.GetString() | |||
} | |||
func (m *Message) GetTypeID() (uint64, error) { | |||
if err := m.GetType(M_ID); err != nil { | |||
return 0, err | |||
} | |||
return m.GetID() | |||
} | |||
func (m *Message) GetTypeSockaddr() (net.Addr, error) { | |||
if err := m.GetType(M_SOCKADDR); err != nil { | |||
return nil, err | |||
} | |||
return m.GetSockaddr() | |||
} | |||
func (m *Message) GetTypeMailaddr() (user, domain string, err error) { | |||
if err = m.GetType(M_MAILADDR); err != nil { | |||
return | |||
} | |||
return m.GetMailaddr() | |||
} | |||
func (m *Message) PutInt(v int) { | |||
var b [4]byte | |||
binary.LittleEndian.PutUint32(b[:], uint32(v)) | |||
m.Data = append(m.Data, b[:]...) | |||
m.Len += 4 | |||
} | |||
func (m *Message) PutUint32(v uint32) { | |||
var b [4]byte | |||
binary.LittleEndian.PutUint32(b[:], v) | |||
m.Data = append(m.Data, b[:]...) | |||
m.Len += 4 | |||
} | |||
func (m *Message) PutString(s string) { | |||
m.Data = append(m.Data, append([]byte(s), 0)...) | |||
m.Len += uint16(len(s)) + 1 | |||
} | |||
func (m *Message) PutID(id uint64) { | |||
var b [8]byte | |||
binary.LittleEndian.PutUint64(b[:], id) | |||
m.Data = append(m.Data, b[:]...) | |||
m.Len += 8 | |||
} | |||
func (m *Message) PutType(t uint8) { | |||
m.Data = append(m.Data, t) | |||
m.Len += 1 | |||
} | |||
func (m *Message) PutTypeInt(v int) { | |||
m.PutType(M_INT) | |||
m.PutInt(v) | |||
} | |||
func (m *Message) PutTypeUint32(v uint32) { | |||
m.PutType(M_UINT32) | |||
m.PutUint32(v) | |||
} | |||
func (m *Message) PutTypeString(s string) { | |||
m.PutType(M_STRING) | |||
m.PutString(s) | |||
} | |||
func (m *Message) PutTypeID(id uint64) { | |||
m.PutType(M_ID) | |||
m.PutID(id) | |||
} |
@@ -0,0 +1,51 @@ | |||
package opensmtpd | |||
import ( | |||
"fmt" | |||
) | |||
const ( | |||
M_INT = iota | |||
M_UINT32 | |||
M_SIZET | |||
M_TIME | |||
M_STRING | |||
M_DATA | |||
M_ID | |||
M_EVPID | |||
M_MSGID | |||
M_SOCKADDR | |||
M_MAILADDR | |||
M_ENVELOPE | |||
) | |||
var mprocTypeName = map[uint8]string{ | |||
M_INT: "M_INT", | |||
M_UINT32: "M_UINT32", | |||
M_SIZET: "M_SIZET", | |||
M_TIME: "M_TIME", | |||
M_STRING: "M_STRING", | |||
M_DATA: "M_DATA", | |||
M_ID: "M_ID", | |||
M_EVPID: "M_EVPID", | |||
M_MSGID: "M_MSGID", | |||
M_SOCKADDR: "M_SOCKADDR", | |||
M_MAILADDR: "M_MAILADDR", | |||
M_ENVELOPE: "M_ENVELOPE", | |||
} | |||
func mprocType(t uint8) string { | |||
if s, ok := mprocTypeName[t]; ok { | |||
return s | |||
} | |||
return fmt.Sprintf("UNKNOWN %d", t) | |||
} | |||
type MProcTypeErr struct { | |||
want, got uint8 | |||
} | |||
func (err MProcTypeErr) Error() string { | |||
return fmt.Sprintf("mproc: expected type %s, got %s", | |||
mprocType(err.want), mprocType(err.got)) | |||
} |
@@ -0,0 +1,15 @@ | |||
package opensmtpd | |||
import ( | |||
"fmt" | |||
"net" | |||
) | |||
type ConnectQuery struct { | |||
Local, Remote net.Addr | |||
Hostname string | |||
} | |||
func (q ConnectQuery) String() string { | |||
return fmt.Sprintf("%s -> %s [hostname=%s]", q.Remote, q.Local, q.Hostname) | |||
} |
@@ -0,0 +1,40 @@ | |||
package opensmtpd | |||
type Session struct { | |||
ID uint64 | |||
filter *Filter | |||
qtype int | |||
qid uint64 | |||
} | |||
func NewSession(f *Filter, id uint64) *Session { | |||
return &Session{ | |||
ID: id, | |||
filter: f, | |||
} | |||
} | |||
func (s *Session) Accept() error { | |||
return s.filter.respond(s, FilterOK, 0, "") | |||
} | |||
func (s *Session) AcceptCode(code int, line string) error { | |||
return s.filter.respond(s, FilterOK, code, line) | |||
} | |||
func (s *Session) Reject(status, code int) error { | |||
if status == FilterOK { | |||
status = FilterFail | |||
} | |||
return s.filter.respond(s, status, code, "") | |||
} | |||
func (s *Session) RejectCode(status, code int, line string) error { | |||
if status == FilterOK { | |||
status = FilterFail | |||
} | |||
return s.filter.respond(s, status, code, line) | |||
} |
@@ -0,0 +1,23 @@ | |||
# Compiled Object files, Static and Dynamic libs (Shared Objects) | |||
*.o | |||
*.a | |||
*.so | |||
# Folders | |||
_obj | |||
_test | |||
# Architecture specific extensions/prefixes | |||
*.[568vq] | |||
[568vq].out | |||
*.cgo1.go | |||
*.cgo2.c | |||
_cgo_defun.c | |||
_cgo_gotypes.go | |||
_cgo_export.* | |||
_testmain.go | |||
*.exe | |||
*.test |
@@ -0,0 +1,212 @@ | |||
package lru | |||
import ( | |||
"fmt" | |||
"sync" | |||
"github.com/hashicorp/golang-lru/simplelru" | |||
) | |||
const ( | |||
// Default2QRecentRatio is the ratio of the 2Q cache dedicated | |||
// to recently added entries that have only been accessed once. | |||
Default2QRecentRatio = 0.25 | |||
// Default2QGhostEntries is the default ratio of ghost | |||
// entries kept to track entries recently evicted | |||
Default2QGhostEntries = 0.50 | |||
) | |||
// TwoQueueCache is a thread-safe fixed size 2Q cache. | |||
// 2Q is an enhancement over the standard LRU cache | |||
// in that it tracks both frequently and recently used | |||
// entries separately. This avoids a burst in access to new | |||
// entries from evicting frequently used entries. It adds some | |||
// additional tracking overhead to the standard LRU cache, and is | |||
// computationally about 2x the cost, and adds some metadata over | |||
// head. The ARCCache is similar, but does not require setting any | |||
// parameters. | |||
type TwoQueueCache struct { | |||
size int | |||
recentSize int | |||
recent *simplelru.LRU | |||
frequent *simplelru.LRU | |||
recentEvict *simplelru.LRU | |||
lock sync.RWMutex | |||
} | |||
// New2Q creates a new TwoQueueCache using the default | |||
// values for the parameters. | |||
func New2Q(size int) (*TwoQueueCache, error) { | |||
return New2QParams(size, Default2QRecentRatio, Default2QGhostEntries) | |||
} | |||
// New2QParams creates a new TwoQueueCache using the provided | |||
// parameter values. | |||
func New2QParams(size int, recentRatio float64, ghostRatio float64) (*TwoQueueCache, error) { | |||
if size <= 0 { | |||
return nil, fmt.Errorf("invalid size") | |||
} | |||
if recentRatio < 0.0 || recentRatio > 1.0 { | |||
return nil, fmt.Errorf("invalid recent ratio") | |||
} | |||
if ghostRatio < 0.0 || ghostRatio > 1.0 { | |||
return nil, fmt.Errorf("invalid ghost ratio") | |||
} | |||
// Determine the sub-sizes | |||
recentSize := int(float64(size) * recentRatio) | |||
evictSize := int(float64(size) * ghostRatio) | |||
// Allocate the LRUs | |||
recent, err := simplelru.NewLRU(size, nil) | |||
if err != nil { | |||
return nil, err | |||
} | |||
frequent, err := simplelru.NewLRU(size, nil) | |||
if err != nil { | |||
return nil, err | |||
} | |||
recentEvict, err := simplelru.NewLRU(evictSize, nil) | |||
if err != nil { | |||
return nil, err | |||
} | |||
// Initialize the cache | |||
c := &TwoQueueCache{ | |||
size: size, | |||
recentSize: recentSize, | |||
recent: recent, | |||
frequent: frequent, | |||
recentEvict: recentEvict, | |||
} | |||
return c, nil | |||
} | |||
func (c *TwoQueueCache) Get(key interface{}) (interface{}, bool) { | |||
c.lock.Lock() | |||
defer c.lock.Unlock() | |||
// Check if this is a frequent value | |||
if val, ok := c.frequent.Get(key); ok { | |||
return val, ok | |||
} | |||
// If the value is contained in recent, then we | |||
// promote it to frequent | |||
if val, ok := c.recent.Peek(key); ok { | |||
c.recent.Remove(key) | |||
c.frequent.Add(key, val) | |||
return val, ok | |||
} | |||
// No hit | |||
return nil, false | |||
} | |||
func (c *TwoQueueCache) Add(key, value interface{}) { | |||
c.lock.Lock() | |||
defer c.lock.Unlock() | |||
// Check if the value is frequently used already, | |||
// and just update the value | |||
if c.frequent.Contains(key) { | |||
c.frequent.Add(key, value) | |||
return | |||
} | |||
// Check if the value is recently used, and promote | |||
// the value into the frequent list | |||
if c.recent.Contains(key) { | |||
c.recent.Remove(key) | |||
c.frequent.Add(key, value) | |||
return | |||
} | |||
// If the value was recently evicted, add it to the | |||
// frequently used list | |||
if c.recentEvict.Contains(key) { | |||
c.ensureSpace(true) | |||
c.recentEvict.Remove(key) | |||
c.frequent.Add(key, value) | |||
return | |||
} | |||
// Add to the recently seen list | |||
c.ensureSpace(false) | |||
c.recent.Add(key, value) | |||
return | |||
} | |||
// ensureSpace is used to ensure we have space in the cache | |||
func (c *TwoQueueCache) ensureSpace(recentEvict bool) { | |||
// If we have space, nothing to do | |||
recentLen := c.recent.Len() | |||
freqLen := c.frequent.Len() | |||
if recentLen+freqLen < c.size { | |||
return | |||
} | |||
// If the recent buffer is larger than | |||
// the target, evict from there | |||
if recentLen > 0 && (recentLen > c.recentSize || (recentLen == c.recentSize && !recentEvict)) { | |||
k, _, _ := c.recent.RemoveOldest() | |||
c.recentEvict.Add(k, nil) | |||
return | |||
} | |||
// Remove from the frequent list otherwise | |||
c.frequent.RemoveOldest() | |||
} | |||
func (c *TwoQueueCache) Len() int { | |||
c.lock.RLock() | |||
defer c.lock.RUnlock() | |||
return c.recent.Len() + c.frequent.Len() | |||
} | |||
func (c *TwoQueueCache) Keys() []interface{} { | |||
c.lock.RLock() | |||
defer c.lock.RUnlock() | |||
k1 := c.frequent.Keys() | |||
k2 := c.recent.Keys() | |||
return append(k1, k2...) | |||
} | |||
func (c *TwoQueueCache) Remove(key interface{}) { | |||
c.lock.Lock() | |||
defer c.lock.Unlock() | |||
if c.frequent.Remove(key) { | |||
return | |||
} | |||
if c.recent.Remove(key) { | |||
return | |||
} | |||
if c.recentEvict.Remove(key) { | |||
return | |||
} | |||
} | |||
func (c *TwoQueueCache) Purge() { | |||
c.lock.Lock() | |||
defer c.lock.Unlock() | |||
c.recent.Purge() | |||
c.frequent.Purge() | |||
c.recentEvict.Purge() | |||
} | |||
func (c *TwoQueueCache) Contains(key interface{}) bool { | |||
c.lock.RLock() | |||
defer c.lock.RUnlock() | |||
return c.frequent.Contains(key) || c.recent.Contains(key) | |||
} | |||
func (c *TwoQueueCache) Peek(key interface{}) (interface{}, bool) { | |||
c.lock.RLock() | |||
defer c.lock.RUnlock() | |||
if val, ok := c.frequent.Peek(key); ok { | |||
return val, ok | |||
} | |||
return c.recent.Peek(key) | |||
} |
@@ -0,0 +1,306 @@ | |||
package lru | |||
import ( | |||
"math/rand" | |||
"testing" | |||
) | |||
func Benchmark2Q_Rand(b *testing.B) { | |||
l, err := New2Q(8192) | |||
if err != nil { | |||
b.Fatalf("err: %v", err) | |||
} | |||
trace := make([]int64, b.N*2) | |||
for i := 0; i < b.N*2; i++ { | |||
trace[i] = rand.Int63() % 32768 | |||
} | |||
b.ResetTimer() | |||
var hit, miss int | |||
for i := 0; i < 2*b.N; i++ { | |||
if i%2 == 0 { | |||
l.Add(trace[i], trace[i]) | |||
} else { | |||
_, ok := l.Get(trace[i]) | |||
if ok { | |||
hit++ | |||
} else { | |||
miss++ | |||
} | |||
} | |||
} | |||
b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss)) | |||
} | |||
func Benchmark2Q_Freq(b *testing.B) { | |||
l, err := New2Q(8192) | |||
if err != nil { | |||
b.Fatalf("err: %v", err) | |||
} | |||
trace := make([]int64, b.N*2) | |||
for i := 0; i < b.N*2; i++ { | |||
if i%2 == 0 { | |||
trace[i] = rand.Int63() % 16384 | |||
} else { | |||
trace[i] = rand.Int63() % 32768 | |||
} | |||
} | |||
b.ResetTimer() | |||
for i := 0; i < b.N; i++ { | |||
l.Add(trace[i], trace[i]) | |||
} | |||
var hit, miss int | |||
for i := 0; i < b.N; i++ { | |||
_, ok := l.Get(trace[i]) | |||
if ok { | |||
hit++ | |||
} else { | |||
miss++ | |||
} | |||
} | |||
b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss)) | |||
} | |||
func Test2Q_RandomOps(t *testing.T) { | |||
size := 128 | |||
l, err := New2Q(128) | |||
if err != nil { | |||
t.Fatalf("err: %v", err) | |||
} | |||
n := 200000 | |||
for i := 0; i < n; i++ { | |||
key := rand.Int63() % 512 | |||
r := rand.Int63() | |||
switch r % 3 { | |||
case 0: | |||
l.Add(key, key) | |||
case 1: | |||
l.Get(key) | |||
case 2: | |||
l.Remove(key) | |||
} | |||
if l.recent.Len()+l.frequent.Len() > size { | |||
t.Fatalf("bad: recent: %d freq: %d", | |||
l.recent.Len(), l.frequent.Len()) | |||
} | |||
} | |||
} | |||
func Test2Q_Get_RecentToFrequent(t *testing.T) { | |||
l, err := New2Q(128) | |||
if err != nil { | |||
t.Fatalf("err: %v", err) | |||
} | |||
// Touch all the entries, should be in t1 | |||
for i := 0; i < 128; i++ { | |||
l.Add(i, i) | |||
} | |||
if n := l.recent.Len(); n != 128 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.frequent.Len(); n != 0 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
// Get should upgrade to t2 | |||
for i := 0; i < 128; i++ { | |||
_, ok := l.Get(i) | |||
if !ok { | |||
t.Fatalf("missing: %d", i) | |||
} | |||
} | |||
if n := l.recent.Len(); n != 0 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.frequent.Len(); n != 128 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
// Get be from t2 | |||
for i := 0; i < 128; i++ { | |||
_, ok := l.Get(i) | |||
if !ok { | |||
t.Fatalf("missing: %d", i) | |||
} | |||
} | |||
if n := l.recent.Len(); n != 0 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.frequent.Len(); n != 128 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
} | |||
func Test2Q_Add_RecentToFrequent(t *testing.T) { | |||
l, err := New2Q(128) | |||
if err != nil { | |||
t.Fatalf("err: %v", err) | |||
} | |||
// Add initially to recent | |||
l.Add(1, 1) | |||
if n := l.recent.Len(); n != 1 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.frequent.Len(); n != 0 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
// Add should upgrade to frequent | |||
l.Add(1, 1) | |||
if n := l.recent.Len(); n != 0 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.frequent.Len(); n != 1 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
// Add should remain in frequent | |||
l.Add(1, 1) | |||
if n := l.recent.Len(); n != 0 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.frequent.Len(); n != 1 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
} | |||
func Test2Q_Add_RecentEvict(t *testing.T) { | |||
l, err := New2Q(4) | |||
if err != nil { | |||
t.Fatalf("err: %v", err) | |||
} | |||
// Add 1,2,3,4,5 -> Evict 1 | |||
l.Add(1, 1) | |||
l.Add(2, 2) | |||
l.Add(3, 3) | |||
l.Add(4, 4) | |||
l.Add(5, 5) | |||
if n := l.recent.Len(); n != 4 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.recentEvict.Len(); n != 1 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.frequent.Len(); n != 0 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
// Pull in the recently evicted | |||
l.Add(1, 1) | |||
if n := l.recent.Len(); n != 3 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.recentEvict.Len(); n != 1 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.frequent.Len(); n != 1 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
// Add 6, should cause another recent evict | |||
l.Add(6, 6) | |||
if n := l.recent.Len(); n != 3 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.recentEvict.Len(); n != 2 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
if n := l.frequent.Len(); n != 1 { | |||
t.Fatalf("bad: %d", n) | |||
} | |||
} | |||
func Test2Q(t *testing.T) { | |||
l, err := New2Q(128) | |||
if err != nil { | |||
t.Fatalf("err: %v", err) | |||
} | |||
for i := 0; i < 256; i++ { | |||
l.Add(i, i) | |||
} | |||
if l.Len() != 128 { | |||
t.Fatalf("bad len: %v", l.Len()) | |||
} | |||
for i, k := range l.Keys() { | |||
if v, ok := l.Get(k); !ok || v != k || v != i+128 { | |||
t.Fatalf("bad key: %v", k) | |||
} | |||
} | |||
for i := 0; i < 128; i++ { | |||
_, ok := l.Get(i) | |||
if ok { | |||
t.Fatalf("should be evicted") | |||
} | |||
} | |||
for i := 128; i < 256; i++ { | |||
_, ok := l.Get(i) | |||
if !ok { | |||
t.Fatalf("should not be evicted") | |||
} | |||
} | |||
for i := 128; i < 192; i++ { | |||
l.Remove(i) | |||
_, ok := l.Get(i) | |||
if ok { | |||
t.Fatalf("should be deleted") | |||
} | |||
} | |||
l.Purge() | |||
if l.Len() != 0 { | |||
t.Fatalf("bad len: %v", l.Len()) | |||
} | |||
if _, ok := l.Get(200); ok { | |||
t.Fatalf("should contain nothing") | |||
} | |||
} | |||
// Test that Contains doesn't update recent-ness | |||
func Test2Q_Contains(t *testing.T) { | |||
l, err := New2Q(2) | |||
if err != nil { | |||
t.Fatalf("err: %v", err) | |||
} | |||
l.Add(1, 1) | |||
l.Add(2, 2) | |||
if !l.Contains(1) { | |||
t.Errorf("1 should be contained") | |||
} | |||
l.Add(3, 3) | |||
if l.Contains(1) { | |||
t.Errorf("Contains should not have updated recent-ness of 1") | |||
} | |||
} | |||
// Test that Peek doesn't update recent-ness | |||
func Test2Q_Peek(t *testing.T) { | |||
l, err := New2Q(2) | |||
if err != nil { | |||
t.Fatalf("err: %v", err) | |||
} | |||
l.Add(1, 1) | |||
l.Add(2, 2) | |||
if v, ok := l.Peek(1); !ok || v != 1 { | |||
t.Errorf("1 should be set to 1: %v, %v", v, ok) | |||
} | |||
l.Add(3, 3) | |||
if l.Contains(1) { | |||
t.Errorf("should not have updated recent-ness of 1") | |||
} | |||
} |
@@ -0,0 +1,362 @@ | |||
Mozilla Public License, version 2.0 | |||
1. Definitions | |||
1.1. "Contributor" | |||
means each individual or legal entity that creates, contributes to the | |||
creation of, or owns Covered Software. | |||
1.2. "Contributor Version" | |||
means the combination of the Contributions of others (if any) used by a | |||
Contributor and that particular Contributor's Contribution. | |||
1.3. "Contribution" | |||
means Covered Software of a particular Contributor. | |||
1.4. "Covered Software" | |||
means Source Code Form to which the initial Contributor has attached the | |||
notice in Exhibit A, the Executable Form of such Source Code Form, and | |||
Modifications of such Source Code Form, in each case including portions | |||
thereof. | |||
1.5. "Incompatible With Secondary Licenses" | |||
means | |||
a. that the initial Contributor has attached the notice described in | |||
Exhibit B to the Covered Software; or | |||
b. that the Covered Software was made available under the terms of | |||
version 1.1 or earlier of the License, but not also under the terms of | |||
a Secondary License. | |||
1.6. "Executable Form" | |||
means any form of the work other than Source Code Form. | |||
1.7. "Larger Work" | |||
means a work that combines Covered Software with other material, in a | |||
separate file or files, that is not Covered Software. | |||
1.8. "License" | |||
means this document. | |||
1.9. "Licensable" | |||
means having the right to grant, to the maximum extent possible, whether | |||
at the time of the initial grant or subsequently, any and all of the | |||
rights conveyed by this License. | |||
1.10. "Modifications" | |||
means any of the following: | |||
a. any file in Source Code Form that results from an addition to, | |||
deletion from, or modification of the contents of Covered Software; or | |||
b. any new file in Source Code Form that contains any Covered Software. | |||
1.11. "Patent Claims" of a Contributor | |||
means any patent claim(s), including without limitation, method, | |||
process, and apparatus claims, in any patent Licensable by such | |||
Contributor that would be infringed, but for the grant of the License, | |||
by the making, using, selling, offering for sale, having made, import, | |||
or transfer of either its Contributions or its Contributor Version. | |||
1.12. "Secondary License" | |||
means either the GNU General Public License, Version 2.0, the GNU Lesser | |||
General Public License, Version 2.1, the GNU Affero General Public | |||
License, Version 3.0, or any later versions of those licenses. | |||
1.13. "Source Code Form" | |||
means the form of the work preferred for making modifications. | |||
1.14. "You" (or "Your") | |||
means an individual or a legal entity exercising rights under this | |||
License. For legal entities, "You" includes any entity that controls, is | |||
controlled by, or is under common control with You. For purposes of this | |||
definition, "control" means (a) the power, direct or indirect, to cause | |||
the direction or management of such entity, whether by contract or | |||
otherwise, or (b) ownership of more than fifty percent (50%) of the | |||
outstanding shares or beneficial ownership of such entity. | |||
2. License Grants and Conditions | |||
2.1. Grants | |||
Each Contributor hereby grants You a world-wide, royalty-free, | |||
non-exclusive license: | |||
a. under intellectual property rights (other than patent or trademark) | |||
Licensable by such Contributor to use, reproduce, make available, | |||
modify, display, perform, distribute, and otherwise exploit its | |||
Contributions, either on an unmodified basis, with Modifications, or | |||
as part of a Larger Work; and | |||
b. under Patent Claims of such Contributor to make, use, sell, offer for | |||
sale, have made, import, and otherwise transfer either its | |||
Contributions or its Contributor Version. | |||
2.2. Effective Date | |||
The licenses granted in Section 2.1 with respect to any Contribution | |||
become effective for each Contribution on the date the Contributor first | |||
distributes such Contribution. | |||
2.3. Limitations on Grant Scope | |||
The licenses granted in this Section 2 are the only rights granted under | |||
this License. No additional rights or licenses will be implied from the | |||
distribution or licensing of Covered Software under this License. | |||
Notwithstanding Section 2.1(b) above, no patent license is granted by a | |||
Contributor: | |||
a. for any code that a Contributor has removed from Covered Software; or | |||
b. for infringements caused by: (i) Your and any other third party's | |||
modifications of Covered Software, or (ii) the combination of its | |||
Contributions with other software (except as part of its Contributor | |||
Version); or | |||
c. under Patent Claims infringed by Covered Software in the absence of | |||
its Contributions. | |||
This License does not grant any rights in the trademarks, service marks, | |||
or logos of any Contributor (except as may be necessary to comply with | |||
the notice requirements in Section 3.4). | |||
2.4. Subsequent Licenses | |||
No Contributor makes additional grants as a result of Your choice to | |||
distribute the Covered Software under a subsequent version of this | |||
License (see Section 10.2) or under the terms of a Secondary License (if | |||
permitted under the terms of Section 3.3). | |||
2.5. Representation | |||
Each Contributor represents that the Contributor believes its | |||
Contributions are its original creation(s) or it has sufficient rights to | |||
grant the rights to its Contributions conveyed by this License. | |||
2.6. Fair Use | |||
This License is not intended to limit any rights You have under | |||
applicable copyright doctrines of fair use, fair dealing, or other | |||
equivalents. | |||
2.7. Conditions | |||
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in | |||
Section 2.1. | |||
3. Responsibilities | |||
3.1. Distribution of Source Form | |||
All distribution of Covered Software in Source Code Form, including any | |||
Modifications that You create or to which You contribute, must be under | |||
the terms of this License. You must inform recipients that the Source | |||
Code Form of the Covered Software is governed by the terms of this | |||
License, and how they can obtain a copy of this License. You may not | |||
attempt to alter or restrict the recipients' rights in the Source Code | |||
Form. | |||
3.2. Distribution of Executable Form | |||
If You distribute Covered Software in Executable Form then: | |||
a. such Covered Software must also be made available in Source Code Form, | |||
as described in Section 3.1, and You must inform recipients of the | |||
Executable Form how they can obtain a copy of such Source Code Form by | |||
reasonable means in a timely manner, at a charge no more than the cost | |||
of distribution to the recipient; and | |||
b. You may distribute such Executable Form under the terms of this | |||
License, or sublicense it under different terms, provided that the | |||
license for the Executable Form does not attempt to limit or alter the | |||
recipients' rights in the Source Code Form under this License. | |||
3.3. Distribution of a Larger Work | |||
You may create and distribute a Larger Work under terms of Your choice, | |||
provided that You also comply with the requirements of this License for | |||
the Covered Software. If the Larger Work is a combination of Covered | |||
Software with a work governed by one or more Secondary Licenses, and the | |||
Covered Software is not Incompatible With Secondary Licenses, this | |||
License permits You to additionally distribute such Covered Software | |||
under the terms of such Secondary License(s), so that the recipient of | |||
the Larger Work may, at their option, further distribute the Covered | |||
Software under the terms of either this License or such Secondary | |||
License(s). | |||
3.4. Notices | |||
You may not remove or alter the substance of any license notices | |||
(including copyright notices, patent notices, disclaimers of warranty, or | |||
limitations of liability) contained within the Source Code Form of the | |||
Covered Software, except that You may alter any license notices to the | |||
extent required to remedy known factual inaccuracies. | |||
3.5. Application of Additional Terms | |||
You may choose to offer, and to charge a fee for, warranty, support, | |||
indemnity or liability obligations to one or more recipients of Covered | |||
Software. However, You may do so only on Your own behalf, and not on | |||
behalf of any Contributor. You must make it absolutely clear that any | |||
such warranty, support, indemnity, or liability obligation is offered by | |||
You alone, and You hereby agree to indemnify every Contributor for any | |||
liability incurred by such Contributor as a result of warranty, support, | |||
indemnity or liability terms You offer. You may include additional | |||
disclaimers of warranty and limitations of liability specific to any | |||
jurisdiction. | |||
4. Inability to Comply Due to Statute or Regulation | |||
If it is impossible for You to comply with any of the terms of this License | |||
with respect to some or all of the Covered Software due to statute, | |||
judicial order, or regulation then You must: (a) comply with the terms of | |||
this License to the maximum extent possible; and (b) describe the | |||
limitations and the code they affect. Such description must be placed in a | |||
text file included with all distributions of the Covered Software under | |||
this License. Except to the extent prohibited by statute or regulation, | |||
such description must be sufficiently detailed for a recipient of ordinary | |||
skill to be able to understand it. | |||
5. Termination | |||
5.1. The rights granted under this License will terminate automatically if You | |||
fail to comply with any of its terms. However, if You become compliant, | |||
then the rights granted under this License from a particular Contributor | |||
are reinstated (a) provisionally, unless and until such Contributor | |||
explicitly and finally terminates Your grants, and (b) on an ongoing | |||
basis, if such Contributor fails to notify You of the non-compliance by | |||
some reasonable means prior to 60 days after You have come back into | |||
compliance. Moreover, Your grants from a particular Contributor are | |||
reinstated on an ongoing basis if such Contributor notifies You of the | |||
non-compliance by some reasonable means, this is the first time You have | |||
received notice of non-compliance with this License from such | |||
Contributor, and You become compliant prior to 30 days after Your receipt | |||
of the notice. | |||
5.2. If You initiate litigation against any entity by asserting a patent | |||
infringement claim (excluding declaratory judgment actions, | |||
counter-claims, and cross-claims) alleging that a Contributor Version | |||
directly or indirectly infringes any patent, then the rights granted to | |||
You by any and all Contributors for the Covered Software under Section | |||
2.1 of this License shall terminate. | |||
5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user | |||
license agreements (excluding distributors and resellers) which have been | |||
validly granted by You or Your distributors under this License prior to | |||
termination shall survive termination. | |||
6. Disclaimer of Warranty | |||
Covered Software is provided under this License on an "as is" basis, | |||
without warranty of any kind, either expressed, implied, or statutory, | |||
including, without limitation, warranties that the Covered Software is free | |||
of defects, merchantable, fit for a particular purpose or non-infringing. | |||
The entire risk as to the quality and performance of the Covered Software | |||
is with You. Should any Covered Software prove defective in any respect, | |||
You (not any Contributor) assume the cost of any necessary servicing, | |||
repair, or correction. This disclaimer of warranty constitutes an essential | |||
part of this License. No use of any Covered Software is authorized under | |||
this License except under this disclaimer. | |||
7. Limitation of Liability | |||
Under no circumstances and under no legal theory, whether tort (including | |||
negligence), contract, or otherwise, shall any Contributor, or anyone who | |||
distributes Covered Software as permitted above, be liable to You for any | |||
direct, indirect, special, incidental, or consequential damages of any | |||
character including, without limitation, damages for lost profits, loss of | |||
goodwill, work stoppage, computer failure or malfunction, or any and all | |||
other commercial damages or losses, even if such party shall have been | |||
informed of the possibility of such damages. This limitation of liability | |||
shall not apply to liability for death or personal injury resulting from | |||
such party's negligence to the extent applicable law prohibits such | |||
limitation. Some jurisdictions do not allow the exclusion or limitation of | |||
incidental or consequential damages, so this exclusion and limitation may | |||
not apply to You. | |||
8. Litigation | |||
Any litigation relating to this License may be brought only in the courts | |||
of a jurisdiction where the defendant maintains its principal place of | |||
business and such litigation shall be governed by laws of that | |||
jurisdiction, without reference to its conflict-of-law provisions. Nothing | |||
in this Section shall prevent a party's ability to bring cross-claims or | |||
counter-claims. | |||
9. Miscellaneous | |||
This License represents the complete agreement concerning the subject | |||
matter hereof. If any provision of this License is held to be | |||
unenforceable, such provision shall be reformed only to the extent | |||
necessary to make it enforceable. Any law or regulation which provides that | |||
the language of a contract shall be construed against the drafter shall not | |||
be used to construe this License against a Contributor. | |||
10. Versions of the License | |||
10.1. New Versions | |||
Mozilla Foundation is the license steward. Except as provided in Section | |||
10.3, no one other than the license steward has the right to modify or | |||
publish new versions of this License. Each version will be given a | |||
distinguishing version number. | |||
10.2. Effect of New Versions | |||
You may distribute the Covered Software under the terms of the version | |||
of the License under which You originally received the Covered Software, | |||
or under the terms of any subsequent version published by the license | |||
steward. | |||
10.3. Modified Versions | |||
If you create software not governed by this License, and you want to | |||
create a new license for such software, you may create and use a | |||
modified version of this License if you rename the license and remove | |||
any references to the name of the license steward (except to note that | |||
such modified license differs from this License). | |||
10.4. Distributing Source Code Form that is Incompatible With Secondary | |||
Licenses If You choose to distribute Source Code Form that is | |||
Incompatible With Secondary Licenses under the terms of this version of | |||
the License, the notice described in Exhibit B of this License must be | |||
attached. | |||
Exhibit A - Source Code Form License Notice | |||
This Source Code Form is subject to the | |||
terms of the Mozilla Public License, v. | |||
2.0. If a copy of the MPL was not | |||
distributed with this file, You can | |||
obtain one at | |||
http://mozilla.org/MPL/2.0/. | |||
If it is not possible or desirable to put the notice in a particular file, | |||
then You may include the notice in a location (such as a LICENSE file in a | |||
relevant directory) where a recipient would be likely to look for such a | |||
notice. | |||
You may add additional accurate notices of copyright ownership. | |||
Exhibit B - "Incompatible With Secondary Licenses" Notice | |||
This Source Code Form is "Incompatible | |||
With Secondary Licenses", as defined by | |||
the Mozilla Public License, v. 2.0. |
@@ -0,0 +1,25 @@ | |||
golang-lru | |||
========== | |||
This provides the `lru` package which implements a fixed-size | |||
thread safe LRU cache. It is based on the cache in Groupcache. | |||
Documentation | |||
============= | |||
Full docs are available on [Godoc](http://godoc.org/github.com/hashicorp/golang-lru) | |||
Example | |||
======= | |||
Using the LRU is very simple: | |||
```go | |||
l, _ := New(128) | |||
for i := 0; i < 256; i++ { | |||
l.Add(i, nil) | |||
} | |||
if l.Len() != 128 { | |||
panic(fmt.Sprintf("bad len: %v", l.Len())) | |||
} | |||
``` |
@@ -0,0 +1,257 @@ | |||
package lru | |||
import ( | |||
"sync" | |||
"github.com/hashicorp/golang-lru/simplelru" | |||
) | |||
// ARCCache is a thread-safe fixed size Adaptive Replacement Cache (ARC). | |||
// ARC is an enhancement over the standard LRU cache in that tracks both | |||
// frequency and recency of use. This avoids a burst in access to new | |||
// entries from evicting the frequently used older entries. It adds some | |||
// additional tracking overhead to a standard LRU cache, computationally | |||
// it is roughly 2x the cost, and the extra memory overhead is linear | |||
// with the size of the cache. ARC has been patented by IBM, but is | |||
// similar to the TwoQueueCache (2Q) which requires setting parameters. | |||
type ARCCache struct { | |||
size int // Size is the total capacity of the cache | |||
p int // P is the dynamic preference towards T1 or T2 | |||
t1 *simplelru.LRU // T1 is the LRU for recently accessed items | |||
b1 *simplelru.LRU // B1 is the LRU for evictions from t1 | |||
t2 *simplelru.LRU // T2 is the LRU for frequently accessed items | |||
b2 *simplelru.LRU // B2 is the LRU for evictions from t2 | |||
lock sync.RWMutex | |||
} | |||
// NewARC creates an ARC of the given size | |||
func NewARC(size int) (*ARCCache, error) { | |||
// Create the sub LRUs | |||
b1, err := simplelru.NewLRU(size, nil) | |||
if err != nil { | |||
return nil, err | |||
} | |||
b2, err := simplelru.NewLRU(size, nil) | |||
if err != nil { | |||
return nil, err | |||
} | |||
t1, err := simplelru.NewLRU(size, nil) | |||
if err != nil { | |||
return nil, err | |||
} | |||
t2, err := simplelru.NewLRU(size, nil) | |||
if err != nil { | |||
return nil, err | |||
} | |||
// Initialize the ARC | |||
c := &ARCCache{ | |||
size: size, | |||
p: 0, | |||
t1: t1, | |||
b1: b1, | |||
t2: t2, | |||
b2: b2, | |||
} | |||
return c, nil | |||
} | |||
// Get looks up a key's value from the cache. | |||
func (c *ARCCache) Get(key interface{}) (interface{}, bool) { | |||
c.lock.Lock() | |||
defer c.lock.Unlock() | |||
// Ff the value is contained in T1 (recent), then | |||
// promote it to T2 (frequent) | |||
if val, ok := c.t1.Peek(key); ok { | |||
c.t1.Remove(key) | |||
c.t2.Add(key, val) | |||
return val, ok | |||
} | |||
// Check if the value is contained in T2 (frequent) | |||
if val, ok := c.t2.Get(key); ok { | |||
return val, ok | |||
} | |||
// No hit | |||
return nil, false | |||
} | |||
// Add adds a value to the cache. | |||
func (c *ARCCache) Add(key, value interface{}) { | |||
c.lock.Lock() | |||
defer c.lock.Unlock() | |||
// Check if the value is contained in T1 (recent), and potentially | |||
// promote it to frequent T2 | |||
if c.t1.Contains(key) { | |||
c.t1.Remove(key) | |||
c.t2.Add(key, value) | |||
return | |||
} | |||
// Check if the value is already in T2 (frequent) and update it | |||
if c.t2.Contains(key) { | |||
c.t2.Add(key, value) | |||
return | |||
} | |||
// Check if this value was recently evicted as part of the | |||
// recently used list | |||
if c.b1.Contains(key) { | |||
// T1 set is too small, increase P appropriately | |||
delta := 1 | |||
b1Len := c.b1.Len() | |||
b2Len := c.b2.Len() | |||
if b2Len > b1Len { | |||
delta = b2Len / b1Len | |||
} | |||
if c.p+delta >= c.size { | |||
c.p = c.size | |||
} else { | |||
c.p += delta | |||
} | |||
// Potentially need to make room in the cache | |||
if c.t1.Len()+c.t2.Len() >= c.size { | |||
c.replace(false) | |||
} | |||
// Remove from B1 | |||
c.b1.Remove(key) | |||
// Add the key to the frequently used list | |||
c.t2.Add(key, value) | |||
return | |||
} | |||
// Check if this value was recently evicted as part of the | |||
// frequently used list | |||
if c.b2.Contains(key) { | |||
// T2 set is too small, decrease P appropriately | |||
delta := 1 | |||
b1Len := c.b1.Len() | |||
b2Len := c.b2.Len() | |||
if b1Len > b2Len { | |||
delta = b1Len / b2Len | |||
} | |||
if delta >= c.p { | |||
c.p = 0 | |||
} else { | |||
c.p -= delta | |||
} | |||
// Potentially need to make room in the cache | |||
if c.t1.Len()+c.t2.Len() >= c.size { | |||
c.replace(true) | |||
} | |||
// Remove from B2 | |||
c.b2.Remove(key) | |||
// Add the key to the frequntly used list | |||
c.t2.Add(key, value) | |||
return | |||
} | |||
// Potentially need to make room in the cache | |||
if c.t1.Len()+c.t2.Len() >= c.size { | |||
c.replace(false) | |||
} | |||
// Keep the size of the ghost buffers trim | |||
if c.b1.Len() > c.size-c.p { | |||
c.b1.RemoveOldest() | |||
} | |||
< |