Initial import

command.go

@@ -0,0 +1,46 @@
+package scrub
+
+import (
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+var commandFlags = map[string][]*regexp.Regexp{
+	"mysql":     {re(`-p(\s?\S+)`), re(`--password(?:[= ])(\S+)`)},
+	"mysqldump": {re(`-p(\s?\S+)`), re(`--password(?:[= ])(\S+)`)},
+}
+
+var CommandScrubber commandScrubber
+
+type commandScrubber struct{}
+
+func (cs commandScrubber) Scrub(s string) string {
+	f := splitAfter(s, []rune(defaultWhitespace))
+	for i, p := range f {
+		base := filepath.Base(strings.TrimSpace(p))
+		if args, ok := commandFlags[base]; ok {
+			return strings.Join(f[:i+1], "") + cs.scrubArgs(strings.Join(f[i+1:], ""), args)
+		}
+	}
+	return s
+}
+
+func (cs commandScrubber) scrubArgs(s string, args []*regexp.Regexp) string {
+	for _, arg := range args {
+		matches := arg.FindStringSubmatch(s)
+		if len(matches) > 0 {
+			for _, match := range matches[1:] {
+				s = strings.Replace(s, match, Replacement, 1)
+			}
+		}
+	}
+	return s
+}
+
+var envNames = []string{
+	"auth",
+	"password",
+	"passwd",
+	"secret",
+}

command_test.go

@@ -0,0 +1,19 @@
+package scrub
+
+import "testing"
+
+func TestCommandScrubber(t *testing.T) {
+	var tests = []struct {
+		Test, Want string
+	}{
+		{"/usr/bin/mysqldump", "/usr/bin/mysqldump"},
+		{"mysqldump -u john -h localhost", "mysqldump -u john -h localhost"},
+		{"mysqldump -u john -p testing -h localhost", "mysqldump -u john -p*redacted* -h localhost"},
+		{"/opt/sap/bin/mysqldump -u john -p testing -h localhost", "/opt/sap/bin/mysqldump -u john -p*redacted* -h localhost"},
+	}
+	for _, test := range tests {
+		if v := CommandScrubber.Scrub(test.Test); v != test.Want {
+			t.Errorf("expected %q, got %q", test.Want, v)
+		}
+	}
+}

entropy.go

@@ -0,0 +1,118 @@
+package scrub
+
+import (
+	"log"
+	"math"
+	"strings"
+)
+
+const (
+	defaultWhitespace = " \t\r\n"
+
+	// defaultEntropyThreshold is chosen to not match most UNIX shell commands, but it does match
+	// passwords with sufficient complexity; use with care!
+	defaultEntropyThreshold = 3.75
+)
+
+type EntropyScrubber struct {
+	Whitespace string
+	Threshold  float64
+}
+
+func (es EntropyScrubber) Scrub(s string) string {
+	var (
+		whitespace = es.Whitespace
+		threshold  = es.Threshold
+	)
+	if whitespace == "" {
+		whitespace = defaultWhitespace
+	}
+	if threshold == 0 {
+		threshold = defaultEntropyThreshold
+	}
+
+	f := splitAfter(s, []rune(whitespace))
+	for i, p := range f {
+		if e := entropy(p); e >= threshold {
+			log.Printf("%q entropy %g >= %g", p, e, threshold)
+			f[i] = Replacement
+		}
+	}
+	return strings.Join(f, "")
+}
+
+// Entropy scrubs all high-entropy strings from s.
+func Entropy(s string) string {
+	return EntropyWithThreshold(s, defaultEntropyThreshold)
+}
+
+// EntropyWithThreshold is like Entropy with a custom threshold.
+func EntropyWithThreshold(s string, threshold float64) string {
+	f := strings.Fields(s)
+	for i, p := range f {
+		if entropy(p) > threshold {
+			f[i] = Replacement
+		}
+	}
+	return strings.Join(f, " ")
+}
+
+// entropy calculates the Shannon entropy of string s.
+func entropy(s string) float64 {
+	size := len([]byte(s))
+	if size == 0 {
+		return 0
+	}
+
+	// Calculate the probabilities for a given byte in string s.
+	var (
+		l = float64(size)
+		m = make(map[byte]float64)
+		f float64
+	)
+	for i := 0; i < size; i++ {
+		m[s[i]]++
+	}
+	for _, c := range m {
+		f += c * math.Log2(c)
+	}
+	return math.Log2(l) - f/l
+}
+
+// idealEntropy calculates the ideal Shannon entropy of a string of length n.
+func idealEntropy(n int) float64 {
+	probability := 1.0 / float64(n)
+	return -1.0 * float64(n) * probability * math.Log(probability) / math.Log(2.0)
+}
+
+func splitAfter(s string, separators []rune) (out []string) {
+	var (
+		j int
+		p = []rune(s)
+		l = len(p)
+	)
+	if l == 0 {
+		return
+	}
+	for i := 0; i < l; i++ {
+		if strings.ContainsRune(string(separators), p[i]) {
+			k := i
+			for k < l && strings.ContainsRune(string(separators), p[k]) {
+				k++
+			}
+			out = append(out, s[j:k])
+			i, j = k, k
+		}
+	}
+	if j > 0 && j < l-1 {
+		out = append(out, s[j:])
+	}
+	if len(out) == 0 {
+		out = append(out, s)
+	}
+	return
+}
+
+func isSpace(c byte) bool {
+	return c == ' ' || c == '\t' || c == '\n' || c == '\r'
+}

entropy_test.go

@@ -0,0 +1,54 @@
+package scrub
+
+import "testing"
+
+func TestEntropyScrubber(t *testing.T) {
+	scrubber := EntropyScrubber{
+		Whitespace: defaultWhitespace,
+		Threshold:  defaultEntropyThreshold,
+	}
+	tests := []struct {
+		Test, Want string
+	}{
+		// No scrubbing:
+		{"ps axufwww", "ps axufwww"},
+		// Secret scrubbing:
+		{"chpasswd root coRrecth0rseba++ery9.23.2007staple$", "chpasswd root *redacted*"},
+	}
+	for _, test := range tests {
+		t.Run(test.Test, func(it *testing.T) {
+			if v := scrubber.Scrub(test.Test); v != test.Want {
+				t.Errorf("expected %q to return %q, got %q", test.Test, test.Want, v)
+			}
+		})
+	}
+}
+
+func TestEntropyShannon(t *testing.T) {
+	var tests = []struct {
+		Test string
+		Want float64
+	}{
+		{"", 0},
+		{"1223334444", 1.8464393446710152},
+		{"password123", 3.2776134368191157},
+		{"Tr0ub4dour&3", 3.2516291673878226},
+		{"correcthorsebatterystaple", 3.363856189774724},
+		{"coRrecth0rseba++ery9.23.2007staple$", 4.229003731107054},
+		{"rWibMFACxAUGZmxhVncy", 4.1219280948873624},
+		{"Ba9ZyWABu99[BK#6MBgbH88Tofv)vs$", 4.413716068381602},
+	}
+	for _, test := range tests {
+		t.Run(test.Test, func(it *testing.T) {
+			if v := entropy(test.Test); !almostEqual(v, test.Want) {
+				it.Errorf("expected %q to return %g, got %g", test.Test, test.Want, v)
+			}
+		})
+	}
+}
+
+const epsilon = 0.00000001
+
+func almostEqual(a, b float64) bool {
+	return (a-b) < epsilon && (b-a) < epsilon
+}

go.mod

@@ -0,0 +1,3 @@
+module maze.io/x/scrub
+
+go 1.13

password.go

@@ -0,0 +1,14 @@
+package scrub
+
+import (
+	"regexp"
+)
+
+const (
+	reCryptHash = `(?s)\$[-0-9a-z]+\$((?:rounds=\d+\$)?(?:` + reBase64 + `+)+\$?)`
+)
+
+var (
+	// CryptHash can scrub hashes in common crypt formats, such as Apache and IANA crypt hashes.
+	CryptHash Scrubber = regexpScrubber{regexp.MustCompile(reCryptHash), false}
+)

password_test.go

@@ -0,0 +1,24 @@
+package scrub
+
+import "testing"
+
+func TestCryptHash(t *testing.T) {
+	var tests = []struct {
+		Test string
+		Want string
+	}{
+		{"$0$testing", "$0$*redacted*"},
+		{`$2a$06$/3OeRpbOf8/l6nPPRdZPp.nRiyYqPobEZGdNRBWihQhiFDh1ws1tu`, `$2a$*redacted*`},
+		{`$6$Np2eF8019ITolL$Q4ZB.EYJdr8nD8OyNPnOTuntLbZXl3YN5r49qtRZDd9JOR.5j1s6zQ7zPekxpVi1WEQ7pYB0AJkHU61Th6Ndf0`, `$6$*redacted*`},
+		{`$5$rounds=80000$wnsT7Yr92oJoP28r$cKhJImk5mfuSKV9b3mumNzlbstFUplKtQXXMo4G6Ep5`, `$5$*redacted*`},
+		{`$sha1$c6218$161d1ac8ab38979c5a31cbaba4a67378e7e60845`, `$sha1$*redacted*`},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Want, func(t *testing.T) {
+			if scrubbed := CryptHash.Scrub(test.Test); scrubbed != test.Want {
+				t.Errorf("expected %q, got %q", test.Want, scrubbed)
+			}
+		})
+	}
+}

pem.go

@@ -0,0 +1,22 @@
+package scrub
+
+import "regexp"
+
+func init() {
+	All = append(All, PEMDHParameters, PEMPrivateKey)
+}
+
+type pemBlockScrubber struct {
+	ScrubHeaders bool
+}
+
+var (
+	PEMDHParameters = regexpScrubber{
+		pattern:     regexp.MustCompile(`(?s)-----BEGIN DH PARAMETERS-----\n([+0-9a-zA-Z/\n]+)\n-----END DH PARAMETERS`),
+		equalLength: true,
+	}
+	PEMPrivateKey = regexpScrubber{
+		pattern:     regexp.MustCompile(`(?s)-----BEGIN (?:[A-Z]+ )PRIVATE KEY-----\n([+0-9a-zA-Z/\n]+)\n-----END (?:[A-Z]+ )PRIVATE KEY`),
+		equalLength: true,
+	}
+)

pem_test.go

@@ -0,0 +1,103 @@
+package scrub
+
+import "testing"
+
+func TestPEMDHParameters(t *testing.T) {
+	testScrubber(t, PEMDHParameters, []testScrubberCase{
+		{Test: testDHParams, Want: wantDHParams},
+	})
+}
+
+func TestPEMPrivateKey(t *testing.T) {
+	testScrubber(t, PEMPrivateKey, []testScrubberCase{
+		{Test: testRSAPrivateKey, Want: wantRSAPrivateKey},
+		{Test: testSSHEd25519Key, Want: wantSSHEd25519Key},
+	})
+}
+
+var (
+	testDHParams = `-----BEGIN DH PARAMETERS-----
+MIGHAoGBANob7KUDV/hqwO2TRlE119oYOxrStVPMaqzuPCnXBX0dw5Yn9lYjoMI9
+6jSberXoK58qVZzeCIfgcUg7+JeTuEvD2zBI9iayb6NlYV6/6z6vKQHmCeGmK1k/
+g1c4LRyWfXDsN94o5oYO7LfyIdZCoOG+mkqO60glbQGBNKMXvPcLAgEC
+-----END DH PARAMETERS-----
+`
+	wantDHParams = `-----BEGIN DH PARAMETERS-----
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted***********************************************
+-----END DH PARAMETERS-----
+`
+	testRSAPrivateKey = `-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAtsR8bBSc4ippQhkAO2iPdpQwxjdVfvmHgISQ6pj9lK+XPOu2
+qLknjMd+WxN04BSLKup+Pu+3hbnhiyFG9wbipXQxbhLKFSelDfeKpYEQApTbF0Ec
+wQ0etVnc0V4mElmmOVN8RIG8HfjXB3NO9lInDvdXoc5IMGEzP9i4wqfGpfxKUkAu
+3Mg8/phziJNFJ7f59FlFJrcjOsfmPCV/KqO4zlGY0tLEd0XJ03+0BY6Vs+D3KyyU
+6OZIWjKHYM5MNnDTdSdEfK21e/ul+EYD77bsTLmbHDN+HU0lgNzWJTigpsZymEtp
+8G3GUsJMBxh8zT/uLIT7vfuuUqnO4fhZxiW0XwIDAQABAoIBACW+QeuX/iX+mCoc
+O54JI8dbJw9oEfHc2gzCU6L+4S85a8Qa8We6hN5fvEpWpEY6N9su/c9FdeLZ1igD
+QUJ2W4vLiQGwQ6dGvqE5w5oWIxZFY1FUEvoTGYpd+moKRVZ4yQkBoqILIKwX3WAe
+geoAYSyIC8LQdLv49rpyqQUZ3L4eGDKS8Y9mFHOsYtuKBJo1EgPg4+2gJxntZMLH
+DrH3ZUYB1UZlggdHyoi77oF0wCvwacdYtRDD5K0opGUduyL5+IkWGklC03Gt1dwC
+7RU/zeaGYbI1nybBgeZvKzSAX2S52ImKQSMFWiLsP1DF7yghN1105rVEo+RzPkJe
+ccgWoUECgYEA6DFHlXXXHTBx2MtclhfTMCq9uICjsOrxG2cyKFJV233ETiISSAum
+A4kIC01stk8ARVWXYBWGyguBEkHA3Y42gCCZoU3SZNZuM0RScfR3iJneCztaCrq4
+5ntvqEfOkRHk8/5tFQ/KH4i4LCOy40w6xQ2tjHJLdjBNDafd/t9J7C8CgYEAyYHf
+IvxXiN2bkRf9mD72Epf06LgBk6Of4xP61vU+Ld4OMcF1TviL9G8xMeeftFRpCqpL
+njGqrGcuuSU3a2PsLTb+1q69t9bSWK2oHX9fj8PgSWN91kRqZKA/QnN1aXv2/ww3
+LtVk8bn9+fZwnBNEMi/l67erWuFFqihY2TVPvtECgYAbwkB3mtXz1GXX8EAKZaDG
+4mU6GI75SK04hHbXoThIfFmqqaIb4OChDZHboA7+IKW8pEXro8cwgn2UzC2djzHu
+0XbsdNxRV91m3aUpoHtl5ldIankSTU5rp5gquyLz7vq7PNCXswKMEJFMHZx2Vhe0
+lTUJVGS3JYEgv8/nd5Rj2wKBgExuYE6K3EDjnZApQ10t9HQVAyYKNT7kv06IU4qZ
+Nt567XNd57rud1ddnZFKQ79IjRcohMoaGJyP/p7nSOAI5Jo50+tmGDvU1bAhHjUi
+DQMgzr/HZwGQrbJBPf1cgdpi1MrkvUGcW098tqLLIOdyP1mx5UnFPs+Xxq7F4v1w
+RTEhAoGBAI3rGEqN5f1N+k4EQf1Op2M+//hhwHpyoryc9W8ed/BF3i22IPQeRjlI
+g9bxjODr2RRt/GOH9WlNd5OkcMkqQfRQ75NNEFNW5I85oWP3yuRCPbav51BB1DrF
+Rlg5XnHMNc5SXbMjTQpUjjNVefoNss6FXYZTEvR8k/0CBjh9MFQH
+-----END RSA PRIVATE KEY-----
+`
+	wantRSAPrivateKey = `-----BEGIN RSA PRIVATE KEY-----
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************************
+*redacted*******************************************
+-----END RSA PRIVATE KEY-----
+`
+	testSSHEd25519Key = `-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACB4nWmwuPMYbVcNVzWhWGsu1CRJ0N9Jq+tDXvP0yG/u/AAAAIiRtuIakbbi
+GgAAAAtzc2gtZWQyNTUxOQAAACB4nWmwuPMYbVcNVzWhWGsu1CRJ0N9Jq+tDXvP0yG/u/A
+AAAECquNjGZMIC/5JAJ3qFpcmmJ439t5iD3XWsK/dXUjP8RHidabC48xhtVw1XNaFYay7U
+JEnQ30mr60Ne8/TIb+78AAAAAAECAwQF
+-----END OPENSSH PRIVATE KEY-----
+`
+	wantSSHEd25519Key = `-----BEGIN OPENSSH PRIVATE KEY-----
+*redacted*************************************************************
+*redacted*************************************************************
+*redacted*************************************************************
+*redacted*************************************************************
+*redacted***********************
+-----END OPENSSH PRIVATE KEY-----
+`
+)

scrub.go

@@ -0,0 +1,122 @@
+// Package scrub offers data scrubbing options for protecting sensitive data.
+package scrub
+
+import (
+	"io"
+	"log"
+	"regexp"
+	"strings"
+)
+
+var (
+	// Replacement string.
+	Replacement = `*redacted*`
+
+	// ReplaceChar is used for equal length replacement.
+	ReplaceChar = '*'
+)
+
+// Common patterns
+const (
+	reBase64 = `[0-9A-Za-z./+=,$-]`
+)
+
+// Scrubber redacts sensitive data.
+type Scrubber interface {
+	// Scrub a string.
+	Scrub(string) string
+}
+
+// Scrubbers are zero or more Scrubber that act as a single scrubber.
+type Scrubbers []Scrubber
+
+// Scrub with all scrubbers, the first scrubber to alter the input will return the scrubbed output.
+func (scrubbers Scrubbers) Scrub(s string) string {
+	for _, scrubber := range scrubbers {
+		if v := scrubber.Scrub(s); v != s {
+			log.Printf("%T %q -> %q", scrubber, s, v)
+			return v
+		}
+	}
+	return s
+}
+
+// All registered scrubbers in safe evaluation order.
+var All = Scrubbers{
+	CommandScrubber,
+	CryptHash,
+	PEMDHParameters,
+	PEMPrivateKey,
+	EntropyScrubber{
+		Whitespace: defaultWhitespace,
+		Threshold:  defaultEntropyThreshold,
+	},
+}
+
+// WrapReader acts as an io.Reader.
+func WrapReader(r io.Reader, scrubbers Scrubbers) io.Reader {
+	if len(scrubbers) == 0 {
+		return r
+	}
+	return reader{r, scrubbers}
+}
+
+// re is shorthand to compile a regular expression
+func re(pattern string) *regexp.Regexp {
+	return regexp.MustCompile(pattern)
+}
+
+// ScrubEqualLength redacts match from in, keeping the same length.
+func scrubEqualLength(in, match string) string {
+	var (
+		l       = len(match)
+		replace = []byte(strings.Repeat(string(ReplaceChar), l))
+	)
+	copy(replace, Replacement)
+
+	// We also keep newline characters from the original string.
+	for i, c := range []byte(match) {
+		if c == '\n' || c == '\r' || c == '\t' {
+			replace[i] = c
+			if i+1 < l {
+				copy(replace[i+1:], Replacement)
+			}
+		}
+	}
+
+	// Replace output.
+	return strings.Replace(in, match, string(replace), 1)
+}
+
+type reader struct {
+	r io.Reader
+	s Scrubbers
+}
+
+func (r reader) Read(p []byte) (n int, err error) {
+	if n, err = r.r.Read(p); n > 0 {
+		for _, s := range r.s {
+			copy(p[:n], s.Scrub(string(p[:n])))
+		}
+	}
+	return
+}
+
+type regexpScrubber struct {
+	pattern     *regexp.Regexp
+	equalLength bool
+}
+
+func (r regexpScrubber) Scrub(s string) string {
+	matches := r.pattern.FindStringSubmatch(s)
+	if len(matches) > 0 {
+		for _, match := range matches[1:] {
+			if r.equalLength {
+				s = scrubEqualLength(s, match)
+			} else {
+				s = strings.Replace(s, match, Replacement, 1)
+			}
+		}
+	}
+	return s
+}

scrub_test.go

@@ -0,0 +1,49 @@
+package scrub
+
+import (
+	"strings"
+	"testing"
+)
+
+type testScrubberCase struct {
+	Test, Want string
+}
+
+func testScrubber(t *testing.T, s Scrubber, tests []testScrubberCase) {
+	t.Helper()
+	for _, test := range tests {
+		t.Run(strings.SplitN(test.Want, "\n", 2)[0], func(it *testing.T) {
+			if result := s.Scrub(test.Test); result != test.Want {
+				it.Fatalf("expected %q, got %q", test.Want, result)
+			}
+		})
+	}
+}
+
+func TestAll(t *testing.T) {
+	testScrubber(t, All, []testScrubberCase{
+		// Command tests.
+		{"/usr/bin/mysqldump", "/usr/bin/mysqldump"},
+		{"mysqldump -u john -h localhost", "mysqldump -u john -h localhost"},
+		{"mysqldump -u john -p testing -h localhost", "mysqldump -u john -p*redacted* -h localhost"},
+		{"/opt/sap/bin/mysqldump -u john -p testing -h localhost", "/opt/sap/bin/mysqldump -u john -p*redacted* -h localhost"},
+
+		// Entropy tests.
+		// No scrubbing:
+		{"ps axufwww", "ps axufwww"},
+		// Secret scrubbing:
+		{"chpasswd root coRrecth0rseba++ery9.23.2007staple$", "chpasswd root *redacted*"},
+
+		// Password tests.
+		{"$0$testing", "$0$*redacted*"},
+		{`$2a$06$/3OeRpbOf8/l6nPPRdZPp.nRiyYqPobEZGdNRBWihQhiFDh1ws1tu`, `$2a$*redacted*`},
+		{`$6$Np2eF8019ITolL$Q4ZB.EYJdr8nD8OyNPnOTuntLbZXl3YN5r49qtRZDd9JOR.5j1s6zQ7zPekxpVi1WEQ7pYB0AJkHU61Th6Ndf0`, `$6$*redacted*`},
+		{`$5$rounds=80000$wnsT7Yr92oJoP28r$cKhJImk5mfuSKV9b3mumNzlbstFUplKtQXXMo4G6Ep5`, `$5$*redacted*`},
+		{`$sha1$c6218$161d1ac8ab38979c5a31cbaba4a67378e7e60845`, `$sha1$*redacted*`},
+
+		// PEM tests.
+		{Test: testDHParams, Want: wantDHParams},
+		{Test: testRSAPrivateKey, Want: wantRSAPrivateKey},
+		{Test: testSSHEd25519Key, Want: wantSSHEd25519Key},
+	})
+}