Browse Source

Initial import into Github

Wijnand 1 year ago
parent
commit
c5f744e4ac

+ 161
- 0
cmd/filter-rbl/main.go View File

@@ -0,0 +1,161 @@
1
+package main
2
+
3
+import (
4
+	"flag"
5
+	"fmt"
6
+	"log"
7
+	"net"
8
+	"os"
9
+	"strings"
10
+
11
+	lru "github.com/hashicorp/golang-lru"
12
+
13
+	"gopkg.in/opensmtpd.v0"
14
+)
15
+
16
+var (
17
+	prog = os.Args[0]
18
+	skip = []*net.IPNet{}
19
+	rbls = []string{
20
+		"b.barracudacentral.org",
21
+		"bl.spamcop.net",
22
+		"virbl.bit.nl",
23
+		"xbl.spamhaus.org",
24
+	}
25
+	debug bool
26
+	masq  bool
27
+	cache *lru.Cache
28
+)
29
+
30
+func debugf(fmt string, args ...interface{}) {
31
+	if !debug {
32
+		return
33
+	}
34
+	log.Printf("debug: "+fmt, args...)
35
+}
36
+
37
+func reverse(ip net.IP) string {
38
+	if ip.To4() == nil {
39
+		return ""
40
+	}
41
+
42
+	splitAddress := strings.Split(ip.String(), ".")
43
+
44
+	for i, j := 0, len(splitAddress)-1; i < len(splitAddress)/2; i, j = i+1, j-1 {
45
+		splitAddress[i], splitAddress[j] = splitAddress[j], splitAddress[i]
46
+	}
47
+
48
+	return strings.Join(splitAddress, ".")
49
+}
50
+
51
+func lookup(rbl string, host string) (result string, listed bool, err error) {
52
+	host = fmt.Sprintf("%s.%s", host, rbl)
53
+
54
+	var res []string
55
+	res, err = net.LookupHost(host)
56
+	if listed = len(res) > 0; listed {
57
+		txt, _ := net.LookupTXT(host)
58
+		if len(txt) > 0 {
59
+			result = txt[0]
60
+		}
61
+	}
62
+
63
+	// Expected error
64
+	if err != nil && strings.HasSuffix(err.Error(), ": no such host") {
65
+		err = nil
66
+	}
67
+
68
+	return
69
+}
70
+
71
+func onConnect(s *opensmtpd.Session, query *opensmtpd.ConnectQuery) error {
72
+	ip := query.Remote.(opensmtpd.Sockaddr).IP()
73
+	if ip == nil {
74
+		return nil
75
+	}
76
+
77
+	debugf("%s: connect from %s\n", prog, ip)
78
+
79
+	for _, ipnet := range skip {
80
+		if ipnet.Contains(ip) {
81
+			debugf("%s: skip %s, IP ignored", prog, ip)
82
+			return s.Accept()
83
+		}
84
+	}
85
+
86
+	var (
87
+		result string
88
+		lsited bool
89
+		host   = reverse(ip)
90
+		err    error
91
+	)
92
+	for _, rbl := range rbls {
93
+		if result, listed, err = lookup(rbl, host); err != nil {
94
+			log.Printf("%s: %s failed %s: %v\n", prog, rbl, ip, err)
95
+		} else if listed {
96
+			log.Printf("%s: %s listed %s: %v\n", prog, rbl, ip, result)
97
+			cache.Add(s.ID, result)
98
+			break
99
+		}
100
+	}
101
+
102
+	debugf("%s: pass: %s\n", prog, ip)
103
+
104
+	if !listed {
105
+		// Add negative hit
106
+		cache.Add(s.ID, "")
107
+	}
108
+
109
+	return s.Accept()
110
+}
111
+
112
+func onDATA(s *opensmtpd.Session) error {
113
+	debugf("%s: %s DATA\n", prog, s)
114
+
115
+	if result, block := cache.Get(s.ID); block && result.(string) != "" {
116
+		return s.RejectCode(opensmtpd.FilterClose, 421, result.(string))
117
+	}
118
+
119
+	return s.Accept()
120
+}
121
+
122
+func main() {
123
+	cacheSize := flag.Int("cache-size", 1024, "LRU cache size")
124
+	rblServer := flag.String("servers", strings.Join(rbls, ","), "RBL servers")
125
+	ignoreIPs := flag.String("ignore", "127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,fe80::/64", "ignore IPs")
126
+	debugging := flag.Bool("d", false, "be verbose")
127
+	verbosity := flag.Bool("v", false, "be verbose")
128
+	flag.BoolVar(&masq, "masq", true, "masquerade SMTP banner")
129
+	flag.Parse()
130
+
131
+	debug = *debugging || *verbosity
132
+
133
+	var err error
134
+	if cache, err = lru.New(*cacheSize); err != nil {
135
+		log.Fatalln(err)
136
+	}
137
+
138
+	rbls = strings.Split(*rblServer, ",")
139
+
140
+	for _, prefix := range strings.Split(*ignoreIPs, ",") {
141
+		var ipnet *net.IPNet
142
+		if _, ipnet, err = net.ParseCIDR(prefix); err != nil {
143
+			log.Fatalln(err)
144
+		}
145
+		skip = append(skip, ipnet)
146
+		debugf("ignore: %s\n", ipnet)
147
+	}
148
+
149
+	f := new(opensmtpd.Filter)
150
+
151
+	f.OnConnect(onConnect)
152
+	f.OnDATA(onDATA)
153
+
154
+	if err = f.Register(); err != nil {
155
+		log.Fatalln(err)
156
+	}
157
+
158
+	if err = f.Serve(); err != nil {
159
+		log.Fatalln(err)
160
+	}
161
+}

+ 11
- 0
conn.go View File

@@ -0,0 +1,11 @@
1
+package opensmtpd
2
+
3
+import (
4
+	"net"
5
+	"os"
6
+)
7
+
8
+func NewConn(fd int) (net.Conn, error) {
9
+	f := os.NewFile(uintptr(fd), "")
10
+	return net.FileConn(f)
11
+}

+ 488
- 0
filter.go View File

@@ -0,0 +1,488 @@
1
+package opensmtpd
2
+
3
+import (
4
+	"fmt"
5
+	"log"
6
+	"net"
7
+	"os"
8
+	"strings"
9
+
10
+	lru "github.com/hashicorp/golang-lru"
11
+)
12
+
13
+const (
14
+	FilterVersion = 51
15
+)
16
+
17
+const (
18
+	TypeFilterRegister uint32 = iota
19
+	TypeFilterEvent
20
+	TypeFilterQuery
21
+	TypeFilterPipe
22
+	TypeFilterResponse
23
+)
24
+
25
+var filterTypeName = map[uint32]string{
26
+	TypeFilterRegister: "IMSG_FILTER_REGISTER",
27
+	TypeFilterEvent:    "IMSG_FILTER_EVENT",
28
+	TypeFilterQuery:    "IMSG_FILTER_QUERY",
29
+	TypeFilterPipe:     "IMSG_FILTER_PIPE",
30
+	TypeFilterResponse: "IMSG_FILTER_RESPONSE",
31
+}
32
+
33
+func filterName(t uint32) string {
34
+	if s, ok := filterTypeName[t]; ok {
35
+		return s
36
+	}
37
+	return fmt.Sprintf("UNKNOWN %d", t)
38
+}
39
+
40
+const (
41
+	HookConnect = 1 << iota
42
+	HookHELO
43
+	HookMAIL
44
+	HookRCPT
45
+	HookDATA
46
+	HookEOM
47
+	HookReset
48
+	HookDisconnect
49
+	HookCommit
50
+	HookRollback
51
+	HookDataLine
52
+)
53
+
54
+var hookTypeName = map[uint16]string{
55
+	HookConnect:    "HOOK_CONNECT",
56
+	HookHELO:       "HOOK_HELO",
57
+	HookMAIL:       "HOOK_MAIL",
58
+	HookRCPT:       "HOOK_RCPT",
59
+	HookDATA:       "HOOK_DATA",
60
+	HookEOM:        "HOOK_EOM",
61
+	HookReset:      "HOOK_RESET",
62
+	HookDisconnect: "HOOK_DISCONNECT",
63
+	HookCommit:     "HOOK_COMMIT",
64
+	HookRollback:   "HOOK_ROLLBACK",
65
+	HookDataLine:   "HOOK_DATALINE",
66
+}
67
+
68
+func hookName(h uint16) string {
69
+	var s []string
70
+	for i := uint(0); i < 11; i++ {
71
+		if h&(1<<i) != 0 {
72
+			s = append(s, hookTypeName[(1<<i)])
73
+		}
74
+	}
75
+	return strings.Join(s, ",")
76
+}
77
+
78
+const (
79
+	EventConnect = iota
80
+	EventReset
81
+	EventDisconnect
82
+	EventTXBegin
83
+	EventTXCommit
84
+	EventTXRollback
85
+)
86
+
87
+var eventTypeName = map[int]string{
88
+	EventConnect:    "EVENT_CONNECT",
89
+	EventReset:      "EVENT_RESET",
90
+	EventDisconnect: "EVENT_DISCONNECT",
91
+	EventTXBegin:    "EVENT_TX_BEGIN",
92
+	EventTXCommit:   "EVENT_TX_COMMIT",
93
+	EventTXRollback: "EVENT_TX_ROLLBACK",
94
+}
95
+
96
+func eventName(t int) string {
97
+	if s, ok := eventTypeName[t]; ok {
98
+		return s
99
+	}
100
+	return fmt.Sprintf("UNKNOWN %d", int(t))
101
+}
102
+
103
+const (
104
+	QueryConnect = iota
105
+	QueryHELO
106
+	QueryMAIL
107
+	QueryRCPT
108
+	QueryDATA
109
+	QueryEOM
110
+	QueryDataLine
111
+)
112
+
113
+var queryTypeName = map[int]string{
114
+	QueryConnect:  "QUERY_CONNECT",
115
+	QueryHELO:     "QUERY_HELO",
116
+	QueryMAIL:     "QUERY_MAIL",
117
+	QueryRCPT:     "QUERY_RCPT",
118
+	QueryDATA:     "QUERY_DATA",
119
+	QueryEOM:      "QUERY_EOM",
120
+	QueryDataLine: "QUERY_DATALINE",
121
+}
122
+
123
+func queryName(t int) string {
124
+	if s, ok := queryTypeName[t]; ok {
125
+		return s
126
+	}
127
+	return fmt.Sprintf("UNKNOWN %d", int(t))
128
+}
129
+
130
+const (
131
+	FilterOK = iota
132
+	FilterFail
133
+	FilterClose
134
+)
135
+
136
+var responseTypeName = map[int]string{
137
+	FilterOK:    "FILTER_OK",
138
+	FilterFail:  "FILTER_FAIL",
139
+	FilterClose: "FILTER_CLOSE",
140
+}
141
+
142
+func responseName(c int) string {
143
+	if s, ok := responseTypeName[c]; ok {
144
+		return s
145
+	}
146
+	return fmt.Sprintf("UNKNOWN %d", c)
147
+}
148
+
149
+type Filter struct {
150
+	Name    string
151
+	Version uint32
152
+
153
+	c net.Conn
154
+	m *Message
155
+
156
+	hooks int
157
+	flags int
158
+	ready bool
159
+
160
+	hook struct {
161
+		connect    func(*Session, *ConnectQuery) error
162
+		helo       func(*Session, string) error
163
+		mail       func(*Session, string, string) error
164
+		rcpt       func(*Session, string, string) error
165
+		data       func(*Session) error
166
+		dataline   func(*Session, string) error
167
+		eom        func(*Session, uint32) error
168
+		reset      func(*Session) error
169
+		disconnect func(*Session) error
170
+		commit     func(*Session) error
171
+	}
172
+	session *lru.Cache
173
+}
174
+
175
+func (f *Filter) OnConnect(fn func(*Session, *ConnectQuery) error) {
176
+	f.hook.connect = fn
177
+	f.hooks |= HookConnect
178
+}
179
+
180
+func (f *Filter) OnHELO(fn func(*Session, string) error) {
181
+	f.hook.helo = fn
182
+	f.hooks |= HookHELO
183
+}
184
+
185
+func (f *Filter) OnMAIL(fn func(*Session, string, string) error) {
186
+	f.hook.mail = fn
187
+	f.hooks |= HookMAIL
188
+}
189
+
190
+func (f *Filter) OnRCPT(fn func(*Session, string, string) error) {
191
+	f.hook.rcpt = fn
192
+	f.hooks |= HookRCPT
193
+}
194
+
195
+func (f *Filter) OnDATA(fn func(*Session) error) {
196
+	f.hook.data = fn
197
+	f.hooks |= HookDATA
198
+}
199
+
200
+func (f *Filter) OnDataLine(fn func(*Session, string) error) {
201
+	f.hook.dataline = fn
202
+	f.hooks |= HookDataLine
203
+}
204
+
205
+func (f *Filter) Register() error {
206
+	var err error
207
+	if f.m == nil {
208
+		f.m = new(Message)
209
+	}
210
+	if f.c == nil {
211
+		if f.c, err = NewConn(0); err != nil {
212
+			return err
213
+		}
214
+	}
215
+	if err = f.m.ReadFrom(f.c); err != nil {
216
+		return err
217
+	}
218
+
219
+	if t, ok := filterTypeName[f.m.Type]; ok {
220
+		log.Printf("filter: imsg %s\n", t)
221
+	} else {
222
+		log.Printf("filter: imsg UNKNOWN %d\n", f.m.Type)
223
+	}
224
+
225
+	switch f.m.Type {
226
+	case TypeFilterRegister:
227
+		var err error
228
+		if f.Version, err = f.m.GetTypeUint32(); err != nil {
229
+			return err
230
+		}
231
+		if f.Name, err = f.m.GetTypeString(); err != nil {
232
+			return err
233
+		}
234
+		log.Printf("register version=%d,name=%q\n", f.Version, f.Name)
235
+
236
+		f.m.reset()
237
+		f.m.Type = TypeFilterRegister
238
+		f.m.PutTypeInt(f.hooks)
239
+		f.m.PutTypeInt(f.flags)
240
+		if err = f.m.SendTo(0); err != nil {
241
+			return err
242
+		}
243
+	default:
244
+		return fmt.Errorf("filter: unexpected imsg type=%s\n", filterTypeName[f.m.Type])
245
+	}
246
+
247
+	return nil
248
+}
249
+
250
+func (f *Filter) Serve() error {
251
+	var err error
252
+	if f.m == nil {
253
+		f.m = new(Message)
254
+	}
255
+	if f.session == nil {
256
+		if f.session, err = lru.New(1024); err != nil {
257
+			return err
258
+		}
259
+	}
260
+	if f.c == nil {
261
+		if f.c, err = NewConn(0); err != nil {
262
+			return err
263
+		}
264
+	}
265
+
266
+	for {
267
+		//log.Printf("fdcount: %d [pid=%d]\n", fdCount(), os.Getpid())
268
+		if err := f.m.ReadFrom(f.c); err != nil {
269
+			if err.Error() != "resource temporarily unavailable" {
270
+				return err
271
+			}
272
+		}
273
+		if err := f.handle(); err != nil {
274
+			return err
275
+		}
276
+	}
277
+}
278
+
279
+func (f *Filter) handle() (err error) {
280
+	if t, ok := filterTypeName[f.m.Type]; ok {
281
+		log.Printf("filter: imsg %s\n", t)
282
+	} else {
283
+		log.Printf("filter: imsg UNKNOWN %d\n", f.m.Type)
284
+	}
285
+
286
+	switch f.m.Type {
287
+	case TypeFilterEvent:
288
+		if err = f.handleEvent(); err != nil {
289
+			return
290
+		}
291
+
292
+	case TypeFilterQuery:
293
+		if err = f.handleQuery(); err != nil {
294
+			return
295
+		}
296
+	}
297
+
298
+	return
299
+}
300
+
301
+func fdCount() int {
302
+	d, err := os.Open("/proc/self/fd")
303
+	if err != nil {
304
+		log.Printf("fdcount open: %v\n", err)
305
+		return -1
306
+	}
307
+	defer d.Close()
308
+	fds, err := d.Readdirnames(-1)
309
+	if err != nil {
310
+		log.Printf("fdcount: %v\n", err)
311
+		return -1
312
+	}
313
+	return len(fds) - 1 // -1 for os.Open...
314
+}
315
+
316
+func (f *Filter) handleEvent() (err error) {
317
+	var (
318
+		id uint64
319
+		t  int
320
+	)
321
+
322
+	if id, err = f.m.GetTypeID(); err != nil {
323
+		return
324
+	}
325
+	if t, err = f.m.GetTypeInt(); err != nil {
326
+		return
327
+	}
328
+
329
+	log.Printf("imsg event: %s [id=%#x]\n", eventName(t), id)
330
+	log.Printf("imsg event data: %q\n", f.m.Data[14:])
331
+	log.Printf("fdcount: %d [pid=%d]\n", fdCount(), os.Getpid())
332
+
333
+	switch t {
334
+	case EventConnect:
335
+		f.session.Add(id, NewSession(f, id))
336
+	case EventDisconnect:
337
+		f.session.Remove(id)
338
+	}
339
+
340
+	return
341
+}
342
+
343
+func (f *Filter) handleQuery() (err error) {
344
+	var (
345
+		id, qid uint64
346
+		t       int
347
+	)
348
+
349
+	if id, err = f.m.GetTypeID(); err != nil {
350
+		return
351
+	}
352
+	if qid, err = f.m.GetTypeID(); err != nil {
353
+		return
354
+	}
355
+	if t, err = f.m.GetTypeInt(); err != nil {
356
+		return
357
+	}
358
+
359
+	log.Printf("imsg query: %s [id=%#x,qid=%#x]\n", queryName(t), id, qid)
360
+	//log.Printf("imsg query data (%d remaining): %q\n", len(f.m.Data[f.m.rpos:]), f.m.Data[f.m.rpos:])
361
+	//log.Printf("fdcount: %d [pid=%d]\n", fdCount(), os.Getpid())
362
+
363
+	var s *Session
364
+	if cached, ok := f.session.Get(id); ok {
365
+		s = cached.(*Session)
366
+	} else {
367
+		s = NewSession(f, id)
368
+		f.session.Add(id, s)
369
+	}
370
+	s.qtype = t
371
+	s.qid = qid
372
+
373
+	switch t {
374
+	case QueryConnect:
375
+		var query ConnectQuery
376
+		if query.Local, err = f.m.GetTypeSockaddr(); err != nil {
377
+			return
378
+		}
379
+		if query.Remote, err = f.m.GetTypeSockaddr(); err != nil {
380
+			return
381
+		}
382
+		if query.Hostname, err = f.m.GetTypeString(); err != nil {
383
+			return
384
+		}
385
+
386
+		log.Printf("query connect: %s\n", query)
387
+		if f.hook.connect != nil {
388
+			return f.hook.connect(s, &query)
389
+		}
390
+
391
+		log.Printf("filter: WARNING: no connect callback\n")
392
+
393
+	case QueryHELO:
394
+		var line string
395
+		if line, err = f.m.GetTypeString(); err != nil {
396
+			return
397
+		}
398
+
399
+		log.Printf("query HELO: %q\n", line)
400
+		if f.hook.helo != nil {
401
+			return f.hook.helo(s, line)
402
+		}
403
+
404
+		log.Printf("filter: WARNING: no HELO callback\n")
405
+		return f.respond(s, FilterOK, 0, "")
406
+
407
+	case QueryMAIL:
408
+		var user, domain string
409
+		if user, domain, err = f.m.GetTypeMailaddr(); err != nil {
410
+			return
411
+		}
412
+
413
+		log.Printf("query MAIL: %s\n", user+"@"+domain)
414
+		if f.hook.mail != nil {
415
+			return f.hook.mail(s, user, domain)
416
+		}
417
+
418
+		log.Printf("filter: WARNING: no MAIL callback\n")
419
+		return f.respond(s, FilterOK, 0, "")
420
+
421
+	case QueryRCPT:
422
+		var user, domain string
423
+		if user, domain, err = f.m.GetTypeMailaddr(); err != nil {
424
+			return
425
+		}
426
+
427
+		log.Printf("query RCPT: %s\n", user+"@"+domain)
428
+		if f.hook.rcpt != nil {
429
+			return f.hook.rcpt(s, user, domain)
430
+		}
431
+
432
+		log.Printf("filter: WARNING: no RCPT callback\n")
433
+		return f.respond(s, FilterOK, 0, "")
434
+
435
+	case QueryDATA:
436
+		if f.hook.data != nil {
437
+			return f.hook.data(s)
438
+		}
439
+
440
+		log.Printf("filter: WARNING: no DATA callback\n")
441
+		return f.respond(s, FilterOK, 0, "")
442
+
443
+	case QueryEOM:
444
+		var dataLen uint32
445
+		if dataLen, err = f.m.GetTypeUint32(); err != nil {
446
+			return
447
+		}
448
+
449
+		if f.hook.eom != nil {
450
+			return f.hook.eom(s, dataLen)
451
+		}
452
+
453
+		log.Printf("filter: WARNING: no EOM callback\n")
454
+		return f.respond(s, FilterOK, 0, "")
455
+	}
456
+
457
+	return
458
+}
459
+
460
+func (f *Filter) respond(s *Session, status, code int, line string) error {
461
+	log.Printf("filter: %s %s [code=%d,line=%q]\n", filterName(TypeFilterResponse), responseName(status), code, line)
462
+
463
+	if s.qtype == QueryEOM {
464
+		// Not implemented
465
+		return nil
466
+	}
467
+
468
+	m := new(Message)
469
+	m.Type = TypeFilterResponse
470
+	m.PutTypeID(s.qid)
471
+	m.PutTypeInt(s.qtype)
472
+	if s.qtype == QueryEOM {
473
+		// Not imlemented
474
+		return nil
475
+	}
476
+	m.PutTypeInt(status)
477
+	m.PutTypeInt(code)
478
+	if line != "" {
479
+		m.PutTypeString(line)
480
+	}
481
+
482
+	if err := m.WriteTo(f.c); err != nil {
483
+		log.Printf("filter: respond failed: %v\n", err)
484
+		return err
485
+	}
486
+
487
+	return nil
488
+}

+ 315
- 0
imsg.go View File

@@ -0,0 +1,315 @@
1
+package opensmtpd
2
+
3
+import (
4
+	"bytes"
5
+	"encoding/binary"
6
+	"errors"
7
+	"fmt"
8
+	"io"
9
+	"net"
10
+	"os"
11
+	"syscall"
12
+)
13
+
14
+const (
15
+	ibufReadSize   = 65535
16
+	imsgMaxSize    = 16384
17
+	imsgHeaderSize = 4 + 2 + 2 + 4 + 4
18
+	imsgVersion    = 14
19
+
20
+	maxLocalPartSize  = (255 + 1)
21
+	maxDomainPartSize = (255 + 1)
22
+)
23
+
24
+type MessageHeader struct {
25
+	Type   uint32
26
+	Len    uint16
27
+	Flags  uint16
28
+	PeerID uint32
29
+	PID    uint32
30
+}
31
+
32
+type Message struct {
33
+	MessageHeader
34
+	Data []byte
35
+
36
+	// rpos is the read position in the current Data
37
+	rpos int
38
+
39
+	// buf is what we read from the socket (and remains)
40
+	buf []byte
41
+}
42
+
43
+func (m *Message) reset() {
44
+	m.Type = 0
45
+	m.Len = 0
46
+	m.Flags = 0
47
+	m.PeerID = imsgVersion
48
+	m.PID = uint32(os.Getpid())
49
+	m.Data = m.Data[:0]
50
+	m.rpos = 0
51
+	m.buf = m.buf[:0]
52
+}
53
+
54
+func (m *Message) ReadFrom(c net.Conn) error {
55
+	m.reset()
56
+
57
+	head := make([]byte, imsgHeaderSize)
58
+	if _, err := c.Read(head); err != nil {
59
+		return err
60
+	}
61
+
62
+	r := bytes.NewBuffer(head)
63
+	if err := binary.Read(r, binary.LittleEndian, &m.MessageHeader); err != nil {
64
+		return err
65
+	}
66
+
67
+	data := make([]byte, m.MessageHeader.Len-imsgHeaderSize)
68
+	if _, err := c.Read(data); err != nil {
69
+		return err
70
+	}
71
+
72
+	m.Data = data
73
+	return nil
74
+}
75
+
76
+func (m *Message) SendTo(fd int) error {
77
+	m.Len = uint16(len(m.Data)) + imsgHeaderSize
78
+
79
+	buf := new(bytes.Buffer)
80
+	//log.Printf("imsg header: %+v\n", m.MessageHeader)
81
+	if err := binary.Write(buf, binary.LittleEndian, &m.MessageHeader); err != nil {
82
+		return err
83
+	}
84
+	buf.Write(m.Data)
85
+	//log.Printf("imsg send: %d / %q\n", buf.Len(), buf.Bytes())
86
+	return syscall.Sendmsg(fd, buf.Bytes(), nil, nil, 0)
87
+}
88
+
89
+func (m *Message) WriteTo(c net.Conn) error {
90
+	m.Len = uint16(len(m.Data)) + imsgHeaderSize
91
+
92
+	buf := new(bytes.Buffer)
93
+	//log.Printf("imsg header: %+v\n", m.MessageHeader)
94
+	if err := binary.Write(buf, binary.LittleEndian, &m.MessageHeader); err != nil {
95
+		return err
96
+	}
97
+	buf.Write(m.Data)
98
+	//log.Printf("imsg send: %d / %q\n", buf.Len(), buf.Bytes())
99
+
100
+	_, err := c.Write(buf.Bytes())
101
+	return err
102
+}
103
+
104
+func (m *Message) GetInt() (int, error) {
105
+	if m.rpos+4 > len(m.Data) {
106
+		return 0, io.ErrShortBuffer
107
+	}
108
+	i := binary.LittleEndian.Uint32(m.Data[m.rpos:])
109
+	m.rpos += 4
110
+	return int(i), nil
111
+}
112
+
113
+func (m *Message) GetUint32() (uint32, error) {
114
+	if m.rpos+4 > len(m.Data) {
115
+		return 0, io.ErrShortBuffer
116
+	}
117
+	u := binary.LittleEndian.Uint32(m.Data[m.rpos:])
118
+	m.rpos += 4
119
+	return u, nil
120
+}
121
+
122
+func (m *Message) GetSize() (uint64, error) {
123
+	if m.rpos+8 > len(m.Data) {
124
+		return 0, io.ErrShortBuffer
125
+	}
126
+	u := binary.LittleEndian.Uint64(m.Data[m.rpos:])
127
+	m.rpos += 8
128
+	return u, nil
129
+}
130
+
131
+func (m *Message) GetString() (string, error) {
132
+	o := bytes.IndexByte(m.Data[m.rpos:], 0)
133
+	if o < 0 {
134
+		return "", errors.New("imsg: string not NULL-terminated")
135
+	}
136
+
137
+	s := string(m.Data[m.rpos : m.rpos+o])
138
+	m.rpos += o
139
+	return s, nil
140
+}
141
+
142
+func (m *Message) GetID() (uint64, error) {
143
+	if m.rpos+8 > len(m.Data) {
144
+		return 0, io.ErrShortBuffer
145
+	}
146
+	u := binary.LittleEndian.Uint64(m.Data[m.rpos:])
147
+	m.rpos += 8
148
+	return u, nil
149
+}
150
+
151
+type Sockaddr []byte
152
+
153
+func (sa Sockaddr) IP() net.IP {
154
+	switch len(sa) {
155
+	case 16: // IPv4, sockaddr_in
156
+		return net.IP(sa[4:8])
157
+	case 28: // IPv6, sockaddr_in6
158
+		return net.IP(sa[8:24])
159
+	default:
160
+		return nil
161
+	}
162
+}
163
+
164
+func (sa Sockaddr) Port() uint16 {
165
+	switch len(sa) {
166
+	case 16: // IPv4, sockaddr_in
167
+		return binary.LittleEndian.Uint16(sa[2:4])
168
+	case 28: // IPv6, sockaddr_in6
169
+		return binary.LittleEndian.Uint16(sa[2:4])
170
+	default:
171
+		return 0
172
+	}
173
+}
174
+
175
+func (sa Sockaddr) Network() string {
176
+	return "bla"
177
+}
178
+
179
+func (sa Sockaddr) String() string {
180
+	return fmt.Sprintf("%s:%d", sa.IP(), sa.Port())
181
+}
182
+
183
+func (m *Message) GetSockaddr() (net.Addr, error) {
184
+	s, err := m.GetSize()
185
+	if err != nil {
186
+		return nil, err
187
+	}
188
+	if m.rpos+int(s) > len(m.Data) {
189
+		return nil, io.ErrShortBuffer
190
+	}
191
+
192
+	a := make(Sockaddr, s)
193
+	copy(a[:], m.Data[m.rpos:])
194
+	m.rpos += int(s)
195
+
196
+	return a, nil
197
+}
198
+
199
+func (m *Message) GetMailaddr() (user, domain string, err error) {
200
+	var buf [maxLocalPartSize + maxDomainPartSize]byte
201
+	if maxLocalPartSize+maxDomainPartSize > len(m.Data[m.rpos:]) {
202
+		return "", "", io.ErrShortBuffer
203
+	}
204
+	copy(buf[:], m.Data[m.rpos:])
205
+	m.rpos += maxLocalPartSize + maxDomainPartSize
206
+	user = string(buf[:maxLocalPartSize])
207
+	domain = string(buf[maxLocalPartSize:])
208
+	return
209
+}
210
+
211
+func (m *Message) GetType(t uint8) error {
212
+	if m.rpos >= len(m.Data) {
213
+		return io.ErrShortBuffer
214
+	}
215
+
216
+	b := m.Data[m.rpos]
217
+	m.rpos++
218
+	if b != t {
219
+		return MProcTypeErr{t, b}
220
+	}
221
+	return nil
222
+}
223
+
224
+func (m *Message) GetTypeInt() (int, error) {
225
+	if err := m.GetType(M_INT); err != nil {
226
+		return 0, err
227
+	}
228
+	return m.GetInt()
229
+}
230
+
231
+func (m *Message) GetTypeUint32() (uint32, error) {
232
+	if err := m.GetType(M_UINT32); err != nil {
233
+		return 0, err
234
+	}
235
+	return m.GetUint32()
236
+}
237
+
238
+func (m *Message) GetTypeString() (string, error) {
239
+	if err := m.GetType(M_STRING); err != nil {
240
+		return "", err
241
+	}
242
+	return m.GetString()
243
+}
244
+
245
+func (m *Message) GetTypeID() (uint64, error) {
246
+	if err := m.GetType(M_ID); err != nil {
247
+		return 0, err
248
+	}
249
+	return m.GetID()
250
+}
251
+
252
+func (m *Message) GetTypeSockaddr() (net.Addr, error) {
253
+	if err := m.GetType(M_SOCKADDR); err != nil {
254
+		return nil, err
255
+	}
256
+	return m.GetSockaddr()
257
+}
258
+
259
+func (m *Message) GetTypeMailaddr() (user, domain string, err error) {
260
+	if err = m.GetType(M_MAILADDR); err != nil {
261
+		return
262
+	}
263
+	return m.GetMailaddr()
264
+}
265
+
266
+func (m *Message) PutInt(v int) {
267
+	var b [4]byte
268
+	binary.LittleEndian.PutUint32(b[:], uint32(v))
269
+	m.Data = append(m.Data, b[:]...)
270
+	m.Len += 4
271
+}
272
+
273
+func (m *Message) PutUint32(v uint32) {
274
+	var b [4]byte
275
+	binary.LittleEndian.PutUint32(b[:], v)
276
+	m.Data = append(m.Data, b[:]...)
277
+	m.Len += 4
278
+}
279
+
280
+func (m *Message) PutString(s string) {
281
+	m.Data = append(m.Data, append([]byte(s), 0)...)
282
+	m.Len += uint16(len(s)) + 1
283
+}
284
+
285
+func (m *Message) PutID(id uint64) {
286
+	var b [8]byte
287
+	binary.LittleEndian.PutUint64(b[:], id)
288
+	m.Data = append(m.Data, b[:]...)
289
+	m.Len += 8
290
+}
291
+
292
+func (m *Message) PutType(t uint8) {
293
+	m.Data = append(m.Data, t)
294
+	m.Len += 1
295
+}
296
+
297
+func (m *Message) PutTypeInt(v int) {
298
+	m.PutType(M_INT)
299
+	m.PutInt(v)
300
+}
301
+
302
+func (m *Message) PutTypeUint32(v uint32) {
303
+	m.PutType(M_UINT32)
304
+	m.PutUint32(v)
305
+}
306
+
307
+func (m *Message) PutTypeString(s string) {
308
+	m.PutType(M_STRING)
309
+	m.PutString(s)
310
+}
311
+
312
+func (m *Message) PutTypeID(id uint64) {
313
+	m.PutType(M_ID)
314
+	m.PutID(id)
315
+}

+ 51
- 0
mproc.go View File

@@ -0,0 +1,51 @@
1
+package opensmtpd
2
+
3
+import (
4
+	"fmt"
5
+)
6
+
7
+const (
8
+	M_INT = iota
9
+	M_UINT32
10
+	M_SIZET
11
+	M_TIME
12
+	M_STRING
13
+	M_DATA
14
+	M_ID
15
+	M_EVPID
16
+	M_MSGID
17
+	M_SOCKADDR
18
+	M_MAILADDR
19
+	M_ENVELOPE
20
+)
21
+
22
+var mprocTypeName = map[uint8]string{
23
+	M_INT:      "M_INT",
24
+	M_UINT32:   "M_UINT32",
25
+	M_SIZET:    "M_SIZET",
26
+	M_TIME:     "M_TIME",
27
+	M_STRING:   "M_STRING",
28
+	M_DATA:     "M_DATA",
29
+	M_ID:       "M_ID",
30
+	M_EVPID:    "M_EVPID",
31
+	M_MSGID:    "M_MSGID",
32
+	M_SOCKADDR: "M_SOCKADDR",
33
+	M_MAILADDR: "M_MAILADDR",
34
+	M_ENVELOPE: "M_ENVELOPE",
35
+}
36
+
37
+func mprocType(t uint8) string {
38
+	if s, ok := mprocTypeName[t]; ok {
39
+		return s
40
+	}
41
+	return fmt.Sprintf("UNKNOWN %d", t)
42
+}
43
+
44
+type MProcTypeErr struct {
45
+	want, got uint8
46
+}
47
+
48
+func (err MProcTypeErr) Error() string {
49
+	return fmt.Sprintf("mproc: expected type %s, got %s",
50
+		mprocType(err.want), mprocType(err.got))
51
+}

+ 15
- 0
query.go View File

@@ -0,0 +1,15 @@
1
+package opensmtpd
2
+
3
+import (
4
+	"fmt"
5
+	"net"
6
+)
7
+
8
+type ConnectQuery struct {
9
+	Local, Remote net.Addr
10
+	Hostname      string
11
+}
12
+
13
+func (q ConnectQuery) String() string {
14
+	return fmt.Sprintf("%s -> %s [hostname=%s]", q.Remote, q.Local, q.Hostname)
15
+}

+ 40
- 0
session.go View File

@@ -0,0 +1,40 @@
1
+package opensmtpd
2
+
3
+type Session struct {
4
+	ID uint64
5
+
6
+	filter *Filter
7
+	qtype  int
8
+	qid    uint64
9
+}
10
+
11
+func NewSession(f *Filter, id uint64) *Session {
12
+	return &Session{
13
+		ID:     id,
14
+		filter: f,
15
+	}
16
+}
17
+
18
+func (s *Session) Accept() error {
19
+	return s.filter.respond(s, FilterOK, 0, "")
20
+}
21
+
22
+func (s *Session) AcceptCode(code int, line string) error {
23
+	return s.filter.respond(s, FilterOK, code, line)
24
+}
25
+
26
+func (s *Session) Reject(status, code int) error {
27
+	if status == FilterOK {
28
+		status = FilterFail
29
+	}
30
+
31
+	return s.filter.respond(s, status, code, "")
32
+}
33
+
34
+func (s *Session) RejectCode(status, code int, line string) error {
35
+	if status == FilterOK {
36
+		status = FilterFail
37
+	}
38
+
39
+	return s.filter.respond(s, status, code, line)
40
+}

+ 23
- 0
vendor/github.com/hashicorp/golang-lru/.gitignore View File

@@ -0,0 +1,23 @@
1
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
2
+*.o
3
+*.a
4
+*.so
5
+
6
+# Folders
7
+_obj
8
+_test
9
+
10
+# Architecture specific extensions/prefixes
11
+*.[568vq]
12
+[568vq].out
13
+
14
+*.cgo1.go
15
+*.cgo2.c
16
+_cgo_defun.c
17
+_cgo_gotypes.go
18
+_cgo_export.*
19
+
20
+_testmain.go
21
+
22
+*.exe
23
+*.test

+ 212
- 0
vendor/github.com/hashicorp/golang-lru/2q.go View File

@@ -0,0 +1,212 @@
1
+package lru
2
+
3
+import (
4
+	"fmt"
5
+	"sync"
6
+
7
+	"github.com/hashicorp/golang-lru/simplelru"
8
+)
9
+
10
+const (
11
+	// Default2QRecentRatio is the ratio of the 2Q cache dedicated
12
+	// to recently added entries that have only been accessed once.
13
+	Default2QRecentRatio = 0.25
14
+
15
+	// Default2QGhostEntries is the default ratio of ghost
16
+	// entries kept to track entries recently evicted
17
+	Default2QGhostEntries = 0.50
18
+)
19
+
20
+// TwoQueueCache is a thread-safe fixed size 2Q cache.
21
+// 2Q is an enhancement over the standard LRU cache
22
+// in that it tracks both frequently and recently used
23
+// entries separately. This avoids a burst in access to new
24
+// entries from evicting frequently used entries. It adds some
25
+// additional tracking overhead to the standard LRU cache, and is
26
+// computationally about 2x the cost, and adds some metadata over
27
+// head. The ARCCache is similar, but does not require setting any
28
+// parameters.
29
+type TwoQueueCache struct {
30
+	size       int
31
+	recentSize int
32
+
33
+	recent      *simplelru.LRU
34
+	frequent    *simplelru.LRU
35
+	recentEvict *simplelru.LRU
36
+	lock        sync.RWMutex
37
+}
38
+
39
+// New2Q creates a new TwoQueueCache using the default
40
+// values for the parameters.
41
+func New2Q(size int) (*TwoQueueCache, error) {
42
+	return New2QParams(size, Default2QRecentRatio, Default2QGhostEntries)
43
+}
44
+
45
+// New2QParams creates a new TwoQueueCache using the provided
46
+// parameter values.
47
+func New2QParams(size int, recentRatio float64, ghostRatio float64) (*TwoQueueCache, error) {
48
+	if size <= 0 {
49
+		return nil, fmt.Errorf("invalid size")
50
+	}
51
+	if recentRatio < 0.0 || recentRatio > 1.0 {
52
+		return nil, fmt.Errorf("invalid recent ratio")
53
+	}
54
+	if ghostRatio < 0.0 || ghostRatio > 1.0 {
55
+		return nil, fmt.Errorf("invalid ghost ratio")
56
+	}
57
+
58
+	// Determine the sub-sizes
59
+	recentSize := int(float64(size) * recentRatio)
60
+	evictSize := int(float64(size) * ghostRatio)
61
+
62
+	// Allocate the LRUs
63
+	recent, err := simplelru.NewLRU(size, nil)
64
+	if err != nil {
65
+		return nil, err
66
+	}
67
+	frequent, err := simplelru.NewLRU(size, nil)
68
+	if err != nil {
69
+		return nil, err
70
+	}
71
+	recentEvict, err := simplelru.NewLRU(evictSize, nil)
72
+	if err != nil {
73
+		return nil, err
74
+	}
75
+
76
+	// Initialize the cache
77
+	c := &TwoQueueCache{
78
+		size:        size,
79
+		recentSize:  recentSize,
80
+		recent:      recent,
81
+		frequent:    frequent,
82
+		recentEvict: recentEvict,
83
+	}
84
+	return c, nil
85
+}
86
+
87
+func (c *TwoQueueCache) Get(key interface{}) (interface{}, bool) {
88
+	c.lock.Lock()
89
+	defer c.lock.Unlock()
90
+
91
+	// Check if this is a frequent value
92
+	if val, ok := c.frequent.Get(key); ok {
93
+		return val, ok
94
+	}
95
+
96
+	// If the value is contained in recent, then we
97
+	// promote it to frequent
98
+	if val, ok := c.recent.Peek(key); ok {
99
+		c.recent.Remove(key)
100
+		c.frequent.Add(key, val)
101
+		return val, ok
102
+	}
103
+
104
+	// No hit
105
+	return nil, false
106
+}
107
+
108
+func (c *TwoQueueCache) Add(key, value interface{}) {
109
+	c.lock.Lock()
110
+	defer c.lock.Unlock()
111
+
112
+	// Check if the value is frequently used already,
113
+	// and just update the value
114
+	if c.frequent.Contains(key) {
115
+		c.frequent.Add(key, value)
116
+		return
117
+	}
118
+
119
+	// Check if the value is recently used, and promote
120
+	// the value into the frequent list
121
+	if c.recent.Contains(key) {
122
+		c.recent.Remove(key)
123
+		c.frequent.Add(key, value)
124
+		return
125
+	}
126
+
127
+	// If the value was recently evicted, add it to the
128
+	// frequently used list
129
+	if c.recentEvict.Contains(key) {
130
+		c.ensureSpace(true)
131
+		c.recentEvict.Remove(key)
132
+		c.frequent.Add(key, value)
133
+		return
134
+	}
135
+
136
+	// Add to the recently seen list
137
+	c.ensureSpace(false)
138
+	c.recent.Add(key, value)
139
+	return
140
+}
141
+
142
+// ensureSpace is used to ensure we have space in the cache
143
+func (c *TwoQueueCache) ensureSpace(recentEvict bool) {
144
+	// If we have space, nothing to do
145
+	recentLen := c.recent.Len()
146
+	freqLen := c.frequent.Len()
147
+	if recentLen+freqLen < c.size {
148
+		return
149
+	}
150
+
151
+	// If the recent buffer is larger than
152
+	// the target, evict from there
153
+	if recentLen > 0 && (recentLen > c.recentSize || (recentLen == c.recentSize && !recentEvict)) {
154
+		k, _, _ := c.recent.RemoveOldest()
155
+		c.recentEvict.Add(k, nil)
156
+		return
157
+	}
158
+
159
+	// Remove from the frequent list otherwise
160
+	c.frequent.RemoveOldest()
161
+}
162
+
163
+func (c *TwoQueueCache) Len() int {
164
+	c.lock.RLock()
165
+	defer c.lock.RUnlock()
166
+	return c.recent.Len() + c.frequent.Len()
167
+}
168
+
169
+func (c *TwoQueueCache) Keys() []interface{} {
170
+	c.lock.RLock()
171
+	defer c.lock.RUnlock()
172
+	k1 := c.frequent.Keys()
173
+	k2 := c.recent.Keys()
174
+	return append(k1, k2...)
175
+}
176
+
177
+func (c *TwoQueueCache) Remove(key interface{}) {
178
+	c.lock.Lock()
179
+	defer c.lock.Unlock()
180
+	if c.frequent.Remove(key) {
181
+		return
182
+	}
183
+	if c.recent.Remove(key) {
184
+		return
185
+	}
186
+	if c.recentEvict.Remove(key) {
187
+		return
188
+	}
189
+}
190
+
191
+func (c *TwoQueueCache) Purge() {
192
+	c.lock.Lock()
193
+	defer c.lock.Unlock()
194
+	c.recent.Purge()
195
+	c.frequent.Purge()
196
+	c.recentEvict.Purge()
197
+}
198
+
199
+func (c *TwoQueueCache) Contains(key interface{}) bool {
200
+	c.lock.RLock()
201
+	defer c.lock.RUnlock()
202
+	return c.frequent.Contains(key) || c.recent.Contains(key)
203
+}
204
+
205
+func (c *TwoQueueCache) Peek(key interface{}) (interface{}, bool) {
206
+	c.lock.RLock()
207
+	defer c.lock.RUnlock()
208
+	if val, ok := c.frequent.Peek(key); ok {
209
+		return val, ok
210
+	}
211
+	return c.recent.Peek(key)
212
+}

+ 306
- 0
vendor/github.com/hashicorp/golang-lru/2q_test.go View File

@@ -0,0 +1,306 @@
1
+package lru
2
+
3
+import (
4
+	"math/rand"
5
+	"testing"
6
+)
7
+
8
+func Benchmark2Q_Rand(b *testing.B) {
9
+	l, err := New2Q(8192)
10
+	if err != nil {
11
+		b.Fatalf("err: %v", err)
12
+	}
13
+
14
+	trace := make([]int64, b.N*2)
15
+	for i := 0; i < b.N*2; i++ {
16
+		trace[i] = rand.Int63() % 32768
17
+	}
18
+
19
+	b.ResetTimer()
20
+
21
+	var hit, miss int
22
+	for i := 0; i < 2*b.N; i++ {
23
+		if i%2 == 0 {
24
+			l.Add(trace[i], trace[i])
25
+		} else {
26
+			_, ok := l.Get(trace[i])
27
+			if ok {
28
+				hit++
29
+			} else {
30
+				miss++
31
+			}
32
+		}
33
+	}
34
+	b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss))
35
+}
36
+
37
+func Benchmark2Q_Freq(b *testing.B) {
38
+	l, err := New2Q(8192)
39
+	if err != nil {
40
+		b.Fatalf("err: %v", err)
41
+	}
42
+
43
+	trace := make([]int64, b.N*2)
44
+	for i := 0; i < b.N*2; i++ {
45
+		if i%2 == 0 {
46
+			trace[i] = rand.Int63() % 16384
47
+		} else {
48
+			trace[i] = rand.Int63() % 32768
49
+		}
50
+	}
51
+
52
+	b.ResetTimer()
53
+
54
+	for i := 0; i < b.N; i++ {
55
+		l.Add(trace[i], trace[i])
56
+	}
57
+	var hit, miss int
58
+	for i := 0; i < b.N; i++ {
59
+		_, ok := l.Get(trace[i])
60
+		if ok {
61
+			hit++
62
+		} else {
63
+			miss++
64
+		}
65
+	}
66
+	b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss))
67
+}
68
+
69
+func Test2Q_RandomOps(t *testing.T) {
70
+	size := 128
71
+	l, err := New2Q(128)
72
+	if err != nil {
73
+		t.Fatalf("err: %v", err)
74
+	}
75
+
76
+	n := 200000
77
+	for i := 0; i < n; i++ {
78
+		key := rand.Int63() % 512
79
+		r := rand.Int63()
80
+		switch r % 3 {
81
+		case 0:
82
+			l.Add(key, key)
83
+		case 1:
84
+			l.Get(key)
85
+		case 2:
86
+			l.Remove(key)
87
+		}
88
+
89
+		if l.recent.Len()+l.frequent.Len() > size {
90
+			t.Fatalf("bad: recent: %d freq: %d",
91
+				l.recent.Len(), l.frequent.Len())
92
+		}
93
+	}
94
+}
95
+
96
+func Test2Q_Get_RecentToFrequent(t *testing.T) {
97
+	l, err := New2Q(128)
98
+	if err != nil {
99
+		t.Fatalf("err: %v", err)
100
+	}
101
+
102
+	// Touch all the entries, should be in t1
103
+	for i := 0; i < 128; i++ {
104
+		l.Add(i, i)
105
+	}
106
+	if n := l.recent.Len(); n != 128 {
107
+		t.Fatalf("bad: %d", n)
108
+	}
109
+	if n := l.frequent.Len(); n != 0 {
110
+		t.Fatalf("bad: %d", n)
111
+	}
112
+
113
+	// Get should upgrade to t2
114
+	for i := 0; i < 128; i++ {
115
+		_, ok := l.Get(i)
116
+		if !ok {
117
+			t.Fatalf("missing: %d", i)
118
+		}
119
+	}
120
+	if n := l.recent.Len(); n != 0 {
121
+		t.Fatalf("bad: %d", n)
122
+	}
123
+	if n := l.frequent.Len(); n != 128 {
124
+		t.Fatalf("bad: %d", n)
125
+	}
126
+
127
+	// Get be from t2
128
+	for i := 0; i < 128; i++ {
129
+		_, ok := l.Get(i)
130
+		if !ok {
131
+			t.Fatalf("missing: %d", i)
132
+		}
133
+	}
134
+	if n := l.recent.Len(); n != 0 {
135
+		t.Fatalf("bad: %d", n)
136
+	}
137
+	if n := l.frequent.Len(); n != 128 {
138
+		t.Fatalf("bad: %d", n)
139
+	}
140
+}
141
+
142
+func Test2Q_Add_RecentToFrequent(t *testing.T) {
143
+	l, err := New2Q(128)
144
+	if err != nil {
145
+		t.Fatalf("err: %v", err)
146
+	}
147
+
148
+	// Add initially to recent
149
+	l.Add(1, 1)
150
+	if n := l.recent.Len(); n != 1 {
151
+		t.Fatalf("bad: %d", n)
152
+	}
153
+	if n := l.frequent.Len(); n != 0 {
154
+		t.Fatalf("bad: %d", n)
155
+	}
156
+
157
+	// Add should upgrade to frequent
158
+	l.Add(1, 1)
159
+	if n := l.recent.Len(); n != 0 {
160
+		t.Fatalf("bad: %d", n)
161
+	}
162
+	if n := l.frequent.Len(); n != 1 {
163
+		t.Fatalf("bad: %d", n)
164
+	}
165
+
166
+	// Add should remain in frequent
167
+	l.Add(1, 1)
168
+	if n := l.recent.Len(); n != 0 {
169
+		t.Fatalf("bad: %d", n)
170
+	}
171
+	if n := l.frequent.Len(); n != 1 {
172
+		t.Fatalf("bad: %d", n)
173
+	}
174
+}
175
+
176
+func Test2Q_Add_RecentEvict(t *testing.T) {
177
+	l, err := New2Q(4)
178
+	if err != nil {
179
+		t.Fatalf("err: %v", err)
180
+	}
181
+
182
+	// Add 1,2,3,4,5 -> Evict 1
183
+	l.Add(1, 1)
184
+	l.Add(2, 2)
185
+	l.Add(3, 3)
186
+	l.Add(4, 4)
187
+	l.Add(5, 5)
188
+	if n := l.recent.Len(); n != 4 {
189
+		t.Fatalf("bad: %d", n)
190
+	}
191
+	if n := l.recentEvict.Len(); n != 1 {
192
+		t.Fatalf("bad: %d", n)
193
+	}
194
+	if n := l.frequent.Len(); n != 0 {
195
+		t.Fatalf("bad: %d", n)
196
+	}
197
+
198
+	// Pull in the recently evicted
199
+	l.Add(1, 1)
200
+	if n := l.recent.Len(); n != 3 {
201
+		t.Fatalf("bad: %d", n)
202
+	}
203
+	if n := l.recentEvict.Len(); n != 1 {
204
+		t.Fatalf("bad: %d", n)
205
+	}
206
+	if n := l.frequent.Len(); n != 1 {
207
+		t.Fatalf("bad: %d", n)
208
+	}
209
+
210
+	// Add 6, should cause another recent evict
211
+	l.Add(6, 6)
212
+	if n := l.recent.Len(); n != 3 {
213
+		t.Fatalf("bad: %d", n)
214
+	}
215
+	if n := l.recentEvict.Len(); n != 2 {
216
+		t.Fatalf("bad: %d", n)
217
+	}
218
+	if n := l.frequent.Len(); n != 1 {
219
+		t.Fatalf("bad: %d", n)
220
+	}
221
+}
222
+
223
+func Test2Q(t *testing.T) {
224
+	l, err := New2Q(128)
225
+	if err != nil {
226
+		t.Fatalf("err: %v", err)
227
+	}
228
+
229
+	for i := 0; i < 256; i++ {
230
+		l.Add(i, i)
231
+	}
232
+	if l.Len() != 128 {
233
+		t.Fatalf("bad len: %v", l.Len())
234
+	}
235
+
236
+	for i, k := range l.Keys() {
237
+		if v, ok := l.Get(k); !ok || v != k || v != i+128 {
238
+			t.Fatalf("bad key: %v", k)
239
+		}
240
+	}
241
+	for i := 0; i < 128; i++ {
242
+		_, ok := l.Get(i)
243
+		if ok {
244
+			t.Fatalf("should be evicted")
245
+		}
246
+	}
247
+	for i := 128; i < 256; i++ {
248
+		_, ok := l.Get(i)
249
+		if !ok {
250
+			t.Fatalf("should not be evicted")
251
+		}
252
+	}
253
+	for i := 128; i < 192; i++ {
254
+		l.Remove(i)
255
+		_, ok := l.Get(i)
256
+		if ok {
257
+			t.Fatalf("should be deleted")
258
+		}
259
+	}
260
+
261
+	l.Purge()
262
+	if l.Len() != 0 {
263
+		t.Fatalf("bad len: %v", l.Len())
264
+	}
265
+	if _, ok := l.Get(200); ok {
266
+		t.Fatalf("should contain nothing")
267
+	}
268
+}
269
+
270
+// Test that Contains doesn't update recent-ness
271
+func Test2Q_Contains(t *testing.T) {
272
+	l, err := New2Q(2)
273
+	if err != nil {
274
+		t.Fatalf("err: %v", err)
275
+	}
276
+
277
+	l.Add(1, 1)
278
+	l.Add(2, 2)
279
+	if !l.Contains(1) {
280
+		t.Errorf("1 should be contained")
281
+	}
282
+
283
+	l.Add(3, 3)
284
+	if l.Contains(1) {
285
+		t.Errorf("Contains should not have updated recent-ness of 1")
286
+	}
287
+}
288
+
289
+// Test that Peek doesn't update recent-ness
290
+func Test2Q_Peek(t *testing.T) {
291
+	l, err := New2Q(2)
292
+	if err != nil {
293
+		t.Fatalf("err: %v", err)
294
+	}
295
+
296
+	l.Add(1, 1)
297
+	l.Add(2, 2)
298
+	if v, ok := l.Peek(1); !ok || v != 1 {
299
+		t.Errorf("1 should be set to 1: %v, %v", v, ok)
300
+	}
301
+
302
+	l.Add(3, 3)
303
+	if l.Contains(1) {
304
+		t.Errorf("should not have updated recent-ness of 1")
305
+	}
306
+}

+ 362
- 0
vendor/github.com/hashicorp/golang-lru/LICENSE View File

@@ -0,0 +1,362 @@
1
+Mozilla Public License, version 2.0
2
+
3
+1. Definitions
4
+
5
+1.1. "Contributor"
6
+
7
+     means each individual or legal entity that creates, contributes to the
8
+     creation of, or owns Covered Software.
9
+
10
+1.2. "Contributor Version"
11
+
12
+     means the combination of the Contributions of others (if any) used by a
13
+     Contributor and that particular Contributor's Contribution.
14
+
15
+1.3. "Contribution"
16
+
17
+     means Covered Software of a particular Contributor.
18
+
19
+1.4. "Covered Software"
20
+
21
+     means Source Code Form to which the initial Contributor has attached the
22
+     notice in Exhibit A, the Executable Form of such Source Code Form, and
23
+     Modifications of such Source Code Form, in each case including portions
24
+     thereof.
25
+
26
+1.5. "Incompatible With Secondary Licenses"
27
+     means
28
+
29
+     a. that the initial Contributor has attached the notice described in
30
+        Exhibit B to the Covered Software; or
31
+
32
+     b. that the Covered Software was made available under the terms of
33
+        version 1.1 or earlier of the License, but not also under the terms of
34
+        a Secondary License.
35
+
36
+1.6. "Executable Form"
37
+
38
+     means any form of the work other than Source Code Form.
39
+
40
+1.7. "Larger Work"
41
+
42
+     means a work that combines Covered Software with other material, in a
43
+     separate file or files, that is not Covered Software.
44
+
45
+1.8. "License"
46
+
47
+     means this document.
48
+
49
+1.9. "Licensable"
50
+
51
+     means having the right to grant, to the maximum extent possible, whether
52
+     at the time of the initial grant or subsequently, any and all of the
53
+     rights conveyed by this License.
54
+
55
+1.10. "Modifications"
56
+
57
+     means any of the following:
58
+
59
+     a. any file in Source Code Form that results from an addition to,
60
+        deletion from, or modification of the contents of Covered Software; or
61
+
62
+     b. any new file in Source Code Form that contains any Covered Software.
63
+
64
+1.11. "Patent Claims" of a Contributor
65
+
66
+      means any patent claim(s), including without limitation, method,
67
+      process, and apparatus claims, in any patent Licensable by such
68
+      Contributor that would be infringed, but for the grant of the License,
69
+      by the making, using, selling, offering for sale, having made, import,
70
+      or transfer of either its Contributions or its Contributor Version.
71
+
72
+1.12. "Secondary License"
73
+
74
+      means either the GNU General Public License, Version 2.0, the GNU Lesser
75
+      General Public License, Version 2.1, the GNU Affero General Public
76
+      License, Version 3.0, or any later versions of those licenses.
77
+
78
+1.13. "Source Code Form"
79
+
80
+      means the form of the work preferred for making modifications.
81
+
82
+1.14. "You" (or "Your")
83
+
84
+      means an individual or a legal entity exercising rights under this
85
+      License. For legal entities, "You" includes any entity that controls, is
86
+      controlled by, or is under common control with You. For purposes of this
87
+      definition, "control" means (a) the power, direct or indirect, to cause
88
+      the direction or management of such entity, whether by contract or
89
+      otherwise, or (b) ownership of more than fifty percent (50%) of the
90
+      outstanding shares or beneficial ownership of such entity.
91
+
92
+
93
+2. License Grants and Conditions
94
+
95
+2.1. Grants
96
+
97
+     Each Contributor hereby grants You a world-wide, royalty-free,
98
+     non-exclusive license:
99
+
100
+     a. under intellectual property rights (other than patent or trademark)
101
+        Licensable by such Contributor to use, reproduce, make available,
102
+        modify, display, perform, distribute, and otherwise exploit its
103
+        Contributions, either on an unmodified basis, with Modifications, or
104
+        as part of a Larger Work; and
105
+
106
+     b. under Patent Claims of such Contributor to make, use, sell, offer for
107
+        sale, have made, import, and otherwise transfer either its
108
+        Contributions or its Contributor Version.
109
+
110
+2.2. Effective Date
111
+
112
+     The licenses granted in Section 2.1 with respect to any Contribution
113
+     become effective for each Contribution on the date the Contributor first
114
+     distributes such Contribution.
115
+
116
+2.3. Limitations on Grant Scope
117
+
118
+     The licenses granted in this Section 2 are the only rights granted under
119
+     this License. No additional rights or licenses will be implied from the
120
+     distribution or licensing of Covered Software under this License.
121
+     Notwithstanding Section 2.1(b) above, no patent license is granted by a
122
+     Contributor:
123
+
124
+     a. for any code that a Contributor has removed from Covered Software; or
125
+
126
+     b. for infringements caused by: (i) Your and any other third party's
127
+        modifications of Covered Software, or (ii) the combination of its
128
+        Contributions with other software (except as part of its Contributor
129
+        Version); or
130
+
131
+     c. under Patent Claims infringed by Covered Software in the absence of
132
+        its Contributions.
133
+
134
+     This License does not grant any rights in the trademarks, service marks,
135
+     or logos of any Contributor (except as may be necessary to comply with
136
+     the notice requirements in Section 3.4).
137
+
138
+2.4. Subsequent Licenses
139
+
140
+     No Contributor makes additional grants as a result of Your choice to
141
+     distribute the Covered Software under a subsequent version of this
142
+     License (see Section 10.2) or under the terms of a Secondary License (if
143
+     permitted under the terms of Section 3.3).
144
+
145
+2.5. Representation
146
+
147
+     Each Contributor represents that the Contributor believes its
148
+     Contributions are its original creation(s) or it has sufficient rights to
149
+     grant the rights to its Contributions conveyed by this License.
150
+
151
+2.6. Fair Use
152
+
153
+     This License is not intended to limit any rights You have under
154
+     applicable copyright doctrines of fair use, fair dealing, or other
155
+     equivalents.
156
+
157
+2.7. Conditions
158
+
159
+     Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
160
+     Section 2.1.
161
+
162
+
163
+3. Responsibilities
164
+
165
+3.1. Distribution of Source Form
166
+
167
+     All distribution of Covered Software in Source Code Form, including any
168
+     Modifications that You create or to which You contribute, must be under
169
+     the terms of this License. You must inform recipients that the Source
170
+     Code Form of the Covered Software is governed by the terms of this
171
+     License, and how they can obtain a copy of this License. You may not
172
+     attempt to alter or restrict the recipients' rights in the Source Code
173
+     Form.
174
+
175
+3.2. Distribution of Executable Form
176
+
177
+     If You distribute Covered Software in Executable Form then:
178
+
179
+     a. such Covered Software must also be made available in Source Code Form,
180
+        as described in Section 3.1, and You must inform recipients of the
181
+        Executable Form how they can obtain a copy of such Source Code Form by
182
+        reasonable means in a timely manner, at a charge no more than the cost
183
+        of distribution to the recipient; and
184
+
185
+     b. You may distribute such Executable Form under the terms of this
186
+        License, or sublicense it under different terms, provided that the
187
+        license for the Executable Form does not attempt to limit or alter the
188
+        recipients' rights in the Source Code Form under this License.
189
+
190
+3.3. Distribution of a Larger Work
191
+
192
+     You may create and distribute a Larger Work under terms of Your choice,
193
+     provided that You also comply with the requirements of this License for
194
+     the Covered Software. If the Larger Work is a combination of Covered
195
+     Software with a work governed by one or more Secondary Licenses, and the
196
+     Covered Software is not Incompatible With Secondary Licenses, this
197
+     License permits You to additionally distribute such Covered Software
198
+     under the terms of such Secondary License(s), so that the recipient of
199
+     the Larger Work may, at their option, further distribute the Covered
200
+     Software under the terms of either this License or such Secondary
201
+     License(s).
202
+
203
+3.4. Notices
204
+
205
+     You may not remove or alter the substance of any license notices
206
+     (including copyright notices, patent notices, disclaimers of warranty, or
207
+     limitations of liability) contained within the Source Code Form of the
208
+     Covered Software, except that You may alter any license notices to the
209
+     extent required to remedy known factual inaccuracies.
210
+
211
+3.5. Application of Additional Terms
212
+
213
+     You may choose to offer, and to charge a fee for, warranty, support,
214
+     indemnity or liability obligations to one or more recipients of Covered
215
+     Software. However, You may do so only on Your own behalf, and not on
216
+     behalf of any Contributor. You must make it absolutely clear that any
217
+     such warranty, support, indemnity, or liability obligation is offered by
218
+     You alone, and You hereby agree to indemnify every Contributor for any
219
+     liability incurred by such Contributor as a result of warranty, support,
220
+     indemnity or liability terms You offer. You may include additional
221
+     disclaimers of warranty and limitations of liability specific to any
222
+     jurisdiction.
223
+
224
+4. Inability to Comply Due to Statute or Regulation
225
+
226
+   If it is impossible for You to comply with any of the terms of this License
227
+   with respect to some or all of the Covered Software due to statute,
228
+   judicial order, or regulation then You must: (a) comply with the terms of
229
+   this License to the maximum extent possible; and (b) describe the
230
+   limitations and the code they affect. Such description must be placed in a
231
+   text file included with all distributions of the Covered Software under
232
+   this License. Except to the extent prohibited by statute or regulation,
233
+   such description must be sufficiently detailed for a recipient of ordinary
234
+   skill to be able to understand it.
235
+
236
+5. Termination
237
+
238
+5.1. The rights granted under this License will terminate automatically if You
239
+     fail to comply with any of its terms. However, if You become compliant,
240
+     then the rights granted under this License from a particular Contributor
241
+     are reinstated (a) provisionally, unless and until such Contributor
242
+     explicitly and finally terminates Your grants, and (b) on an ongoing
243
+     basis, if such Contributor fails to notify You of the non-compliance by
244
+     some reasonable means prior to 60 days after You have come back into
245
+     compliance. Moreover, Your grants from a particular Contributor are
246
+     reinstated on an ongoing basis if such Contributor notifies You of the
247
+     non-compliance by some reasonable means, this is the first time You have
248
+     received notice of non-compliance with this License from such
249
+     Contributor, and You become compliant prior to 30 days after Your receipt
250
+     of the notice.
251
+
252
+5.2. If You initiate litigation against any entity by asserting a patent
253
+     infringement claim (excluding declaratory judgment actions,
254
+     counter-claims, and cross-claims) alleging that a Contributor Version
255
+     directly or indirectly infringes any patent, then the rights granted to
256
+     You by any and all Contributors for the Covered Software under Section
257
+     2.1 of this License shall terminate.
258
+
259
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
260
+     license agreements (excluding distributors and resellers) which have been
261
+     validly granted by You or Your distributors under this License prior to
262
+     termination shall survive termination.
263
+
264
+6. Disclaimer of Warranty
265
+
266
+   Covered Software is provided under this License on an "as is" basis,
267
+   without warranty of any kind, either expressed, implied, or statutory,
268
+   including, without limitation, warranties that the Covered Software is free
269
+   of defects, merchantable, fit for a particular purpose or non-infringing.
270
+   The entire risk as to the quality and performance of the Covered Software
271
+   is with You. Should any Covered Software prove defective in any respect,
272
+   You (not any Contributor) assume the cost of any necessary servicing,
273
+   repair, or correction. This disclaimer of warranty constitutes an essential
274
+   part of this License. No use of  any Covered Software is authorized under
275
+   this License except under this disclaimer.
276
+
277
+7. Limitation of Liability
278
+
279
+   Under no circumstances and under no legal theory, whether tort (including
280
+   negligence), contract, or otherwise, shall any Contributor, or anyone who
281
+   distributes Covered Software as permitted above, be liable to You for any
282
+   direct, indirect, special, incidental, or consequential damages of any
283
+   character including, without limitation, damages for lost profits, loss of
284
+   goodwill, work stoppage, computer failure or malfunction, or any and all
285
+   other commercial damages or losses, even if such party shall have been
286
+   informed of the possibility of such damages. This limitation of liability
287
+   shall not apply to liability for death or personal injury resulting from
288
+   such party's negligence to the extent applicable law prohibits such
289
+   limitation. Some jurisdictions do not allow the exclusion or limitation of
290
+   incidental or consequential damages, so this exclusion and limitation may
291
+   not apply to You.
292
+
293
+8. Litigation
294
+
295
+   Any litigation relating to this License may be brought only in the courts
296
+   of a jurisdiction where the defendant maintains its principal place of
297
+   business and such litigation shall be governed by laws of that
298
+   jurisdiction, without reference to its conflict-of-law provisions. Nothing
299
+   in this Section shall prevent a party's ability to bring cross-claims or
300
+   counter-claims.
301
+
302
+9. Miscellaneous
303
+
304
+   This License represents the complete agreement concerning the subject
305
+   matter hereof. If any provision of this License is held to be
306
+   unenforceable, such provision shall be reformed only to the extent
307
+   necessary to make it enforceable. Any law or regulation which provides that
308
+   the language of a contract shall be construed against the drafter shall not
309
+   be used to construe this License against a Contributor.
310
+
311
+
312
+10. Versions of the License
313
+
314
+10.1. New Versions
315
+
316
+      Mozilla Foundation is the license steward. Except as provided in Section
317
+      10.3, no one other than the license steward has the right to modify or
318
+      publish new versions of this License. Each version will be given a
319
+      distinguishing version number.
320
+
321
+10.2. Effect of New Versions
322
+
323
+      You may distribute the Covered Software under the terms of the version
324
+      of the License under which You originally received the Covered Software,
325
+      or under the terms of any subsequent version published by the license
326
+      steward.
327
+
328
+10.3. Modified Versions
329
+
330
+      If you create software not governed by this License, and you want to
331
+      create a new license for such software, you may create and use a
332
+      modified version of this License if you rename the license and remove
333
+      any references to the name of the license steward (except to note that
334
+      such modified license differs from this License).
335
+
336
+10.4. Distributing Source Code Form that is Incompatible With Secondary
337
+      Licenses If You choose to distribute Source Code Form that is
338
+      Incompatible With Secondary Licenses under the terms of this version of
339
+      the License, the notice described in Exhibit B of this License must be
340
+      attached.
341
+
342
+Exhibit A - Source Code Form License Notice
343
+
344
+      This Source Code Form is subject to the
345
+      terms of the Mozilla Public License, v.
346
+      2.0. If a copy of the MPL was not
347
+      distributed with this file, You can
348
+      obtain one at
349
+      http://mozilla.org/MPL/2.0/.
350
+
351
+If it is not possible or desirable to put the notice in a particular file,
352
+then You may include the notice in a location (such as a LICENSE file in a
353
+relevant directory) where a recipient would be likely to look for such a
354
+notice.
355
+
356
+You may add additional accurate notices of copyright ownership.
357
+
358
+Exhibit B - "Incompatible With Secondary Licenses" Notice
359
+
360
+      This Source Code Form is "Incompatible
361
+      With Secondary Licenses", as defined by
362
+      the Mozilla Public License, v. 2.0.

+ 25
- 0
vendor/github.com/hashicorp/golang-lru/README.md View File

@@ -0,0 +1,25 @@
1
+golang-lru
2
+==========
3
+
4
+This provides the `lru` package which implements a fixed-size
5
+thread safe LRU cache. It is based on the cache in Groupcache.
6
+
7
+Documentation
8
+=============
9
+
10
+Full docs are available on [Godoc](http://godoc.org/github.com/hashicorp/golang-lru)
11
+
12
+Example
13
+=======
14
+
15
+Using the LRU is very simple:
16
+
17
+```go
18
+l, _ := New(128)
19
+for i := 0; i < 256; i++ {
20
+    l.Add(i, nil)
21
+}
22
+if l.Len() != 128 {
23
+    panic(fmt.Sprintf("bad len: %v", l.Len()))
24
+}
25
+```

+ 257
- 0
vendor/github.com/hashicorp/golang-lru/arc.go View File

@@ -0,0 +1,257 @@
1
+package lru
2
+
3
+import (
4
+	"sync"
5
+
6
+	"github.com/hashicorp/golang-lru/simplelru"
7
+)
8
+
9
+// ARCCache is a thread-safe fixed size Adaptive Replacement Cache (ARC).
10
+// ARC is an enhancement over the standard LRU cache in that tracks both
11
+// frequency and recency of use. This avoids a burst in access to new
12
+// entries from evicting the frequently used older entries. It adds some
13
+// additional tracking overhead to a standard LRU cache, computationally
14
+// it is roughly 2x the cost, and the extra memory overhead is linear
15
+// with the size of the cache. ARC has been patented by IBM, but is
16
+// similar to the TwoQueueCache (2Q) which requires setting parameters.
17
+type ARCCache struct {
18
+	size int // Size is the total capacity of the cache
19
+	p    int // P is the dynamic preference towards T1 or T2
20
+
21
+	t1 *simplelru.LRU // T1 is the LRU for recently accessed items
22
+	b1 *simplelru.LRU // B1 is the LRU for evictions from t1
23
+
24
+	t2 *simplelru.LRU // T2 is the LRU for frequently accessed items
25
+	b2 *simplelru.LRU // B2 is the LRU for evictions from t2
26
+
27
+	lock sync.RWMutex
28
+}
29
+
30
+// NewARC creates an ARC of the given size
31
+func NewARC(size int) (*ARCCache, error) {
32
+	// Create the sub LRUs
33
+	b1, err := simplelru.NewLRU(size, nil)
34
+	if err != nil {
35
+		return nil, err
36
+	}
37
+	b2, err := simplelru.NewLRU(size, nil)
38
+	if err != nil {
39
+		return nil, err
40
+	}
41
+	t1, err := simplelru.NewLRU(size, nil)
42
+	if err != nil {
43
+		return nil, err
44
+	}
45
+	t2, err := simplelru.NewLRU(size, nil)
46
+	if err != nil {
47
+		return nil, err
48
+	}
49
+
50
+	// Initialize the ARC
51
+	c := &ARCCache{
52
+		size: size,
53
+		p:    0,
54
+		t1:   t1,
55
+		b1:   b1,
56
+		t2:   t2,
57
+		b2:   b2,
58
+	}
59
+	return c, nil
60
+}
61
+
62
+// Get looks up a key's value from the cache.
63
+func (c *ARCCache) Get(key interface{}) (interface{}, bool) {
64
+	c.lock.Lock()
65
+	defer c.lock.Unlock()
66
+
67
+	// Ff the value is contained in T1 (recent), then
68
+	// promote it to T2 (frequent)
69
+	if val, ok := c.t1.Peek(key); ok {
70
+		c.t1.Remove(key)
71
+		c.t2.Add(key, val)
72
+		return val, ok
73
+	}
74
+
75
+	// Check if the value is contained in T2 (frequent)
76
+	if val, ok := c.t2.Get(key); ok {
77
+		return val, ok
78
+	}
79
+
80
+	// No hit
81
+	return nil, false
82
+}
83
+
84
+// Add adds a value to the cache.
85
+func (c *ARCCache) Add(key, value interface{}) {
86
+	c.lock.Lock()
87
+	defer c.lock.Unlock()
88
+
89
+	// Check if the value is contained in T1 (recent), and potentially
90
+	// promote it to frequent T2
91
+	if c.t1.Contains(key) {
92
+		c.t1.Remove(key)
93
+		c.t2.Add(key, value)
94
+		return
95
+	}
96
+
97
+	// Check if the value is already in T2 (frequent) and update it
98
+	if c.t2.Contains(key) {
99
+		c.t2.Add(key, value)
100
+		return
101
+	}
102
+
103
+	// Check if this value was recently evicted as part of the
104
+	// recently used list
105
+	if c.b1.Contains(key) {
106
+		// T1 set is too small, increase P appropriately
107
+		delta := 1
108
+		b1Len := c.b1.Len()
109
+		b2Len := c.b2.Len()
110
+		if b2Len > b1Len {
111
+			delta = b2Len / b1Len
112
+		}
113
+		if c.p+delta >= c.size {
114
+			c.p = c.size
115
+		} else {
116
+			c.p += delta
117
+		}
118
+
119
+		// Potentially need to make room in the cache
120
+		if c.t1.Len()+c.t2.Len() >= c.size {
121
+			c.replace(false)
122
+		}
123
+
124
+		// Remove from B1
125
+		c.b1.Remove(key)
126
+
127
+		// Add the key to the frequently used list
128
+		c.t2.Add(key, value)
129
+		return
130
+	}
131
+
132
+	// Check if this value was recently evicted as part of the
133
+	// frequently used list
134
+	if c.b2.Contains(key) {
135
+		// T2 set is too small, decrease P appropriately
136
+		delta := 1
137
+		b1Len := c.b1.Len()
138
+		b2Len := c.b2.Len()
139
+		if b1Len > b2Len {
140
+			delta = b1Len / b2Len
141
+		}
142
+		if delta >= c.p {
143
+			c.p = 0
144
+		} else {
145
+			c.p -= delta
146
+		}
147
+
148
+		// Potentially need to make room in the cache
149
+		if c.t1.Len()+c.t2.Len() >= c.size {
150
+			c.replace(true)
151
+		}
152
+
153
+		// Remove from B2
154
+		c.b2.Remove(key)
155
+
156
+		// Add the key to the frequntly used list
157
+		c.t2.Add(key, value)
158
+		return
159
+	}
160
+
161
+	// Potentially need to make room in the cache
162
+	if c.t1.Len()+c.t2.Len() >= c.size {
163
+		c.replace(false)
164
+	}
165
+
166
+	// Keep the size of the ghost buffers trim
167
+	if c.b1.Len() > c.size-c.p {
168
+		c.b1.RemoveOldest()
169
+	}
170
+	if c.b2.Len() > c.p {
171
+		c.b2.RemoveOldest()
172
+	}
173
+
174
+	// Add to the recently seen list
175
+	c.t1.Add(key, value)
176
+	return
177
+}
178
+
179
+// replace is used to adaptively evict from either T1 or T2
180
+// based on the current learned value of P
181
+func (c *ARCCache) replace(b2ContainsKey bool) {
182
+	t1Len := c.t1.Len()
183
+	if t1Len > 0 && (t1Len > c.p || (t1Len == c.p && b2ContainsKey)) {
184
+		k, _, ok := c.t1.RemoveOldest()
185
+		if ok {
186
+			c.b1.Add(k, nil)
187
+		}
188
+	} else {
189
+		k, _, ok := c.t2.RemoveOldest()
190
+		if ok {
191
+			c.b2.Add(k, nil)
192
+		}
193
+	}
194
+}
195
+
196
+// Len returns the number of cached entries
197
+func (c *ARCCache) Len() int {
198
+	c.lock.RLock()
199
+	defer c.lock.RUnlock()
200
+	return c.t1.Len() + c.t2.Len()
201
+}
202
+
203
+// Keys returns all the cached keys
204
+func (c *ARCCache) Keys() []interface{} {
205
+	c.lock.RLock()
206
+	defer c.lock.RUnlock()
207
+	k1 := c.t1.Keys()
208
+	k2 := c.t2.Keys()
209
+	return append(k1, k2...)
210
+}
211
+
212
+// Remove is used to purge a key from the cache
213
+func (c *ARCCache) Remove(key interface{}) {
214
+	c.lock.Lock()
215
+	defer c.lock.Unlock()
216
+	if c.t1.Remove(key) {
217
+		return
218
+	}
219
+	if c.t2.Remove(key) {
220
+		return
221
+	}
222
+	if c.b1.Remove(key) {
223
+		return
224
+	}
225
+	if c.b2.Remove(key) {
226
+		return
227
+	}
228
+}
229
+
230
+// Purge is used to clear the cache
231
+func (c *ARCCache) Purge() {
232
+	c.lock.Lock()
233
+	defer c.lock.Unlock()
234
+	c.t1.Purge()
235
+	c.t2.Purge()
236
+	c.b1.Purge()
237
+	c.b2.Purge()
238
+}
239
+
240
+// Contains is used to check if the cache contains a key
241
+// without updating recency or frequency.
242
+func (c *ARCCache) Contains(key interface{}) bool {
243
+	c.lock.RLock()
244
+	defer c.lock.RUnlock()
245
+	return c.t1.Contains(key) || c.t2.Contains(key)
246
+}
247
+
248
+// Peek is used to inspect the cache value of a key
249
+// without updating recency or frequency.
250
+func (c *ARCCache) Peek(key interface{}) (interface{}, bool) {
251
+	c.lock.RLock()
252
+	defer c.lock.RUnlock()
253
+	if val, ok := c.t1.Peek(key); ok {
254
+		return val, ok
255
+	}
256
+	return c.t2.Peek(key)
257
+}

+ 377
- 0
vendor/github.com/hashicorp/golang-lru/arc_test.go View File

@@ -0,0 +1,377 @@
1
+package lru
2
+
3
+import (
4
+	"math/rand"
5
+	"testing"
6
+	"time"
7
+)
8
+
9
+func init() {
10
+	rand.Seed(time.Now().Unix())
11
+}
12
+
13
+func BenchmarkARC_Rand(b *testing.B) {
14
+	l, err := NewARC(8192)
15
+	if err != nil {
16
+		b.Fatalf("err: %v", err)
17
+	}
18
+
19
+	trace := make([]int64, b.N*2)
20
+	for i := 0; i < b.N*2; i++ {
21
+		trace[i] = rand.Int63() % 32768
22
+	}
23
+
24
+	b.ResetTimer()
25
+
26
+	var hit, miss int
27
+	for i := 0; i < 2*b.N; i++ {
28
+		if i%2 == 0 {
29
+			l.Add(trace[i], trace[i])
30
+		} else {
31
+			_, ok := l.Get(trace[i])
32
+			if ok {
33
+				hit++
34
+			} else {
35
+				miss++
36
+			}
37
+		}
38
+	}
39
+	b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss))
40
+}
41
+
42
+func BenchmarkARC_Freq(b *testing.B) {
43
+	l, err := NewARC(8192)
44
+	if err != nil {
45
+		b.Fatalf("err: %v", err)
46
+	}
47
+
48
+	trace := make([]int64, b.N*2)
49
+	for i := 0; i < b.N*2; i++ {
50
+		if i%2 == 0 {
51
+			trace[i] = rand.Int63() % 16384
52
+		} else {
53
+			trace[i] = rand.Int63() % 32768
54
+		}
55
+	}
56
+
57
+	b.ResetTimer()
58
+
59
+	for i := 0; i < b.N; i++ {
60
+		l.Add(trace[i], trace[i])
61
+	}
62
+	var hit, miss int
63
+	for i := 0; i < b.N; i++ {
64
+		_, ok := l.Get(trace[i])
65
+		if ok {
66
+			hit++
67
+		} else {
68
+			miss++
69
+		}
70
+	}
71
+	b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss))
72
+}
73
+
74
+func TestARC_RandomOps(t *testing.T) {
75
+	size := 128
76
+	l, err := NewARC(128)
77
+	if err != nil {
78
+		t.Fatalf("err: %v", err)
79
+	}
80
+
81
+	n := 200000
82
+	for i := 0; i < n; i++ {
83
+		key := rand.Int63() % 512
84
+		r := rand.Int63()
85
+		switch r % 3 {
86
+		case 0:
87
+			l.Add(key, key)
88
+		case 1:
89
+			l.Get(key)
90
+		case 2:
91
+			l.Remove(key)
92
+		}
93
+
94
+		if l.t1.Len()+l.t2.Len() > size {
95
+			t.Fatalf("bad: t1: %d t2: %d b1: %d b2: %d p: %d",
96
+				l.t1.Len(), l.t2.Len(), l.b1.Len(), l.b2.Len(), l.p)
97
+		}
98
+		if l.b1.Len()+l.b2.Len() > size {
99
+			t.Fatalf("bad: t1: %d t2: %d b1: %d b2: %d p: %d",
100
+				l.t1.Len(), l.t2.Len(), l.b1.Len(), l.b2.Len(), l.p)
101
+		}
102
+	}
103
+}
104
+
105
+func TestARC_Get_RecentToFrequent(t *testing.T) {
106
+	l, err := NewARC(128)
107
+	if err != nil {
108
+		t.Fatalf("err: %v", err)
109
+	}
110
+
111
+	// Touch all the entries, should be in t1
112
+	for i := 0; i < 128; i++ {
113
+		l.Add(i, i)
114
+	}
115
+	if n := l.t1.Len(); n != 128 {
116
+		t.Fatalf("bad: %d", n)
117
+	}
118
+	if n := l.t2.Len(); n != 0 {
119
+		t.Fatalf("bad: %d", n)
120
+	}
121
+
122
+	// Get should upgrade to t2
123
+	for i := 0; i < 128; i++ {
124
+		_, ok := l.Get(i)
125
+		if !ok {
126
+			t.Fatalf("missing: %d", i)
127
+		}
128
+	}
129
+	if n := l.t1.Len(); n != 0 {
130
+		t.Fatalf("bad: %d", n)
131
+	}
132
+	if n := l.t2.Len(); n != 128 {
133
+		t.Fatalf("bad: %d", n)
134
+	}
135
+
136
+	// Get be from t2
137
+	for i := 0; i < 128; i++ {
138
+		_, ok := l.Get(i)
139
+		if !ok {
140
+			t.Fatalf("missing: %d", i)
141
+		}
142
+	}
143
+	if n := l.t1.Len(); n != 0 {
144
+		t.Fatalf("bad: %d", n)
145
+	}
146
+	if n := l.t2.Len(); n != 128 {
147
+		t.Fatalf("bad: %d", n)
148
+	}
149
+}
150
+
151
+func TestARC_Add_RecentToFrequent(t *testing.T) {
152
+	l, err := NewARC(128)
153
+	if err != nil {
154
+		t.Fatalf("err: %v", err)
155
+	}
156
+
157
+	// Add initially to t1
158
+	l.Add(1, 1)
159
+	if n := l.t1.Len(); n != 1 {
160
+		t.Fatalf("bad: %d", n)
161
+	}
162
+	if n := l.t2.Len(); n != 0 {
163
+		t.Fatalf("bad: %d", n)
164
+	}
165
+
166
+	// Add should upgrade to t2
167
+	l.Add(1, 1)
168
+	if n := l.t1.Len(); n != 0 {
169
+		t.Fatalf("bad: %d", n)
170
+	}
171
+	if n := l.t2.Len(); n != 1 {
172
+		t.Fatalf("bad: %d", n)
173
+	}
174
+
175
+	// Add should remain in t2
176
+	l.Add(1, 1)
177
+	if n := l.t1.Len(); n != 0 {
178
+		t.Fatalf("bad: %d", n)
179
+	}
180
+	if n := l.t2.Len(); n != 1 {
181
+		t.Fatalf("bad: %d", n)
182
+	}
183
+}
184
+
185
+func TestARC_Adaptive(t *testing.T) {
186
+	l, err := NewARC(4)
187
+	if err != nil {
188
+		t.Fatalf("err: %v", err)
189
+	}
190
+
191
+	// Fill t1
192
+	for i := 0; i < 4; i++ {
193
+		l.Add(i, i)
194
+	}
195
+	if n := l.t1.Len(); n != 4 {
196
+		t.Fatalf("bad: %d", n)
197
+	}
198
+
199
+	// Move to t2
200
+	l.Get(0)
201
+	l.Get(1)
202
+	if n := l.t2.Len(); n != 2 {
203
+		t.Fatalf("bad: %d", n)
204
+	}
205
+
206
+	// Evict from t1
207
+	l.Add(4, 4)
208
+	if n := l.b1.Len(); n != 1 {
209
+		t.Fatalf("bad: %d", n)
210
+	}
211
+
212
+	// Current state
213
+	// t1 : (MRU) [4, 3] (LRU)
214
+	// t2 : (MRU) [1, 0] (LRU)
215
+	// b1 : (MRU) [2] (LRU)
216
+	// b2 : (MRU) [] (LRU)
217
+
218
+	// Add 2, should cause hit on b1
219
+	l.Add(2, 2)
220
+	if n := l.b1.Len(); n != 1 {
221
+		t.Fatalf("bad: %d", n)
222
+	}
223
+	if l.p != 1 {
224
+		t.Fatalf("bad: %d", l.p)
225
+	}
226
+	if n := l.t2.Len(); n != 3 {
227
+		t.Fatalf("bad: %d", n)
228
+	}
229
+
230
+	// Current state
231
+	// t1 : (MRU) [4] (LRU)
232
+	// t2 : (MRU) [2, 1, 0] (LRU)
233
+	// b1 : (MRU) [3] (LRU)
234
+	// b2 : (MRU) [] (LRU)
235
+
236
+	// Add 4, should migrate to t2
237
+	l.Add(4, 4)
238
+	if n := l.t1.Len(); n != 0 {
239
+		t.Fatalf("bad: %d", n)
240
+	}
241
+	if n := l.t2.Len(); n != 4 {
242
+		t.Fatalf("bad: %d", n)
243
+	}
244
+
245
+	// Current state
246
+	// t1 : (MRU) [] (LRU)
247
+	// t2 : (MRU) [4, 2, 1, 0] (LRU)
248
+	// b1 : (MRU) [3] (LRU)
249
+	// b2 : (MRU) [] (LRU)
250
+
251
+	// Add 4, should evict to b2
252
+	l.Add(5, 5)
253
+	if n := l.t1.Len(); n != 1 {
254
+		t.Fatalf("bad: %d", n)
255
+	}
256
+	if n := l.t2.Len(); n != 3 {
257
+		t.Fatalf("bad: %d", n)
258
+	}
259
+	if n := l.b2.Len(); n != 1 {
260
+		t.Fatalf("bad: %d", n)
261
+	}
262
+
263
+	// Current state
264
+	// t1 : (MRU) [5] (LRU)
265
+	// t2 : (MRU) [4, 2, 1] (LRU)
266
+	// b1 : (MRU) [3] (LRU)
267
+	// b2 : (MRU) [0] (LRU)
268
+
269
+	// Add 0, should decrease p
270
+	l.Add(0, 0)
271
+	if n := l.t1.Len(); n != 0 {
272
+		t.Fatalf("bad: %d", n)
273
+	}
274
+	if n := l.t2.Len(); n != 4 {
275
+		t.Fatalf("bad: %d", n)
276
+	}
277
+	if n := l.b1.Len(); n != 2 {
278
+		t.Fatalf("bad: %d", n)
279
+	}
280
+	if n := l.b2.Len(); n != 0 {
281
+		t.Fatalf("bad: %d", n)
282
+	}
283
+	if l.p != 0 {
284
+		t.Fatalf("bad: %d", l.p)
285
+	}
286
+
287
+	// Current state
288
+	// t1 : (MRU) [] (LRU)
289
+	// t2 : (MRU) [0, 4, 2, 1] (LRU)
290
+	// b1 : (MRU) [5, 3] (LRU)
291
+	// b2 : (MRU) [0] (LRU)
292
+}
293
+
294
+func TestARC(t *testing.T) {
295
+	l, err := NewARC(128)
296
+	if err != nil {
297
+		t.Fatalf("err: %v", err)
298
+	}
299
+
300
+	for i := 0; i < 256; i++ {
301
+		l.Add(i, i)
302
+	}
303
+	if l.Len() != 128 {
304
+		t.Fatalf("bad len: %v", l.Len())
305
+	}
306
+
307
+	for i, k := range l.Keys() {
308
+		if v, ok := l.Get(k); !ok || v != k || v != i+128 {
309
+			t.Fatalf("bad key: %v", k)
310
+		}
311
+	}
312
+	for i := 0; i < 128; i++ {
313
+		_, ok := l.Get(i)
314
+		if ok {
315
+			t.Fatalf("should be evicted")
316
+		}
317
+	}
318
+	for i := 128; i < 256; i++ {
319
+		_, ok := l.Get(i)
320
+		if !ok {
321
+			t.Fatalf("should not be evicted")
322
+		}
323
+	}
324
+	for i := 128; i < 192; i++ {
325
+		l.Remove(i)
326
+		_, ok := l.Get(i)
327
+		if ok {
328
+			t.Fatalf("should be deleted")
329
+		}
330
+	}
331
+
332
+	l.Purge()
333
+	if l.Len() != 0 {
334
+		t.Fatalf("bad len: %v", l.Len())
335
+	}
336
+	if _, ok := l.Get(200); ok {
337
+		t.Fatalf("should contain nothing")
338
+	}
339
+}
340
+
341
+// Test that Contains doesn't update recent-ness
342
+func TestARC_Contains(t *testing.T) {
343
+	l, err := NewARC(2)
344
+	if err != nil {
345
+		t.Fatalf("err: %v", err)
346
+	}
347
+
348
+	l.Add(1, 1)
349
+	l.Add(2, 2)
350
+	if !l.Contains(1) {
351
+		t.Errorf("1 should be contained")
352
+	}
353
+
354
+	l.Add(3, 3)
355
+	if l.Contains(1) {
356
+		t.Errorf("Contains should not have updated recent-ness of 1")
357
+	}
358
+}
359
+
360
+// Test that Peek doesn't update recent-ness
361
+func TestARC_Peek(t *testing.T) {
362
+	l, err := NewARC(2)
363
+	if err != nil {
364
+		t.Fatalf("err: %v", err)
365
+	}
366
+
367
+	l.Add(1, 1)
368
+	l.Add(2, 2)
369
+	if v, ok := l.Peek(1); !ok || v != 1 {
370
+		t.Errorf("1 should be set to 1: %v, %v", v, ok)
371
+	}
372
+
373
+	l.Add(3, 3)
374
+	if l.Contains(1) {
375
+		t.Errorf("should not have updated recent-ness of 1")
376
+	}
377
+}

+ 114
- 0
vendor/github.com/hashicorp/golang-lru/lru.go View File

@@ -0,0 +1,114 @@
1
+// This package provides a simple LRU cache. It is based on the
2
+// LRU implementation in groupcache:
3
+// https://github.com/golang/groupcache/tree/master/lru
4
+package lru
5
+
6
+import (
7
+	"sync"
8
+
9
+	"github.com/hashicorp/golang-lru/simplelru"
10
+)
11
+
12
+// Cache is a thread-safe fixed size LRU cache.
13
+type Cache struct {
14
+	lru  *simplelru.LRU
15
+	lock sync.RWMutex
16
+}
17
+
18
+// New creates an LRU of the given size
19
+func New(size int) (*Cache, error) {
20
+	return NewWithEvict(size, nil)
21
+}
22
+
23
+// NewWithEvict constructs a fixed size cache with the given eviction
24
+// callback.
25
+func NewWithEvict(size int, onEvicted func(key interface{}, value interface{})) (*Cache, error) {
26
+	lru, err := simplelru.NewLRU(size, simplelru.EvictCallback(onEvicted))
27
+	if err != nil {
28
+		return nil, err
29
+	}
30
+	c := &Cache{
31
+		lru: lru,
32
+	}
33
+	return c, nil
34
+}
35
+
36
+// Purge is used to completely clear the cache
37
+func (c *Cache) Purge() {