Browse Source

Initial import

master
maze 1 year ago
parent
commit
5144cdb051
11 changed files with 574 additions and 0 deletions
  1. +46
    -0
      command.go
  2. +19
    -0
      command_test.go
  3. +118
    -0
      entropy.go
  4. +54
    -0
      entropy_test.go
  5. +3
    -0
      go.mod
  6. +14
    -0
      password.go
  7. +24
    -0
      password_test.go
  8. +22
    -0
      pem.go
  9. +103
    -0
      pem_test.go
  10. +122
    -0
      scrub.go
  11. +49
    -0
      scrub_test.go

+ 46
- 0
command.go View File

@ -0,0 +1,46 @@
package scrub
import (
"path/filepath"
"regexp"
"strings"
)
var commandFlags = map[string][]*regexp.Regexp{
"mysql": {re(`-p(\s?\S+)`), re(`--password(?:[= ])(\S+)`)},
"mysqldump": {re(`-p(\s?\S+)`), re(`--password(?:[= ])(\S+)`)},
}
var CommandScrubber commandScrubber
type commandScrubber struct{}
func (cs commandScrubber) Scrub(s string) string {
f := splitAfter(s, []rune(defaultWhitespace))
for i, p := range f {
base := filepath.Base(strings.TrimSpace(p))
if args, ok := commandFlags[base]; ok {
return strings.Join(f[:i+1], "") + cs.scrubArgs(strings.Join(f[i+1:], ""), args)
}
}
return s
}
func (cs commandScrubber) scrubArgs(s string, args []*regexp.Regexp) string {
for _, arg := range args {
matches := arg.FindStringSubmatch(s)
if len(matches) > 0 {
for _, match := range matches[1:] {
s = strings.Replace(s, match, Replacement, 1)
}
}
}
return s
}
var envNames = []string{
"auth",
"password",
"passwd",
"secret",
}

+ 19
- 0
command_test.go View File

@ -0,0 +1,19 @@
package scrub
import "testing"
func TestCommandScrubber(t *testing.T) {
var tests = []struct {
Test, Want string
}{
{"/usr/bin/mysqldump", "/usr/bin/mysqldump"},
{"mysqldump -u john -h localhost", "mysqldump -u john -h localhost"},
{"mysqldump -u john -p testing -h localhost", "mysqldump -u john -p*redacted* -h localhost"},
{"/opt/sap/bin/mysqldump -u john -p testing -h localhost", "/opt/sap/bin/mysqldump -u john -p*redacted* -h localhost"},
}
for _, test := range tests {
if v := CommandScrubber.Scrub(test.Test); v != test.Want {
t.Errorf("expected %q, got %q", test.Want, v)
}
}
}

+ 118
- 0
entropy.go View File

@ -0,0 +1,118 @@
package scrub
import (
"log"
"math"
"strings"
)
const (
defaultWhitespace = " \t\r\n"
// defaultEntropyThreshold is chosen to not match most UNIX shell commands, but it does match
// passwords with sufficient complexity; use with care!
defaultEntropyThreshold = 3.75
)
type EntropyScrubber struct {
Whitespace string
Threshold float64
}
func (es EntropyScrubber) Scrub(s string) string {
var (
whitespace = es.Whitespace
threshold = es.Threshold
)
if whitespace == "" {
whitespace = defaultWhitespace
}
if threshold == 0 {
threshold = defaultEntropyThreshold
}
f := splitAfter(s, []rune(whitespace))
for i, p := range f {
if e := entropy(p); e >= threshold {
log.Printf("%q entropy %g >= %g", p, e, threshold)
f[i] = Replacement
}
}
return strings.Join(f, "")
}
// Entropy scrubs all high-entropy strings from s.
func Entropy(s string) string {
return EntropyWithThreshold(s, defaultEntropyThreshold)
}
// EntropyWithThreshold is like Entropy with a custom threshold.
func EntropyWithThreshold(s string, threshold float64) string {
f := strings.Fields(s)
for i, p := range f {
if entropy(p) > threshold {
f[i] = Replacement
}
}
return strings.Join(f, " ")
}
// entropy calculates the Shannon entropy of string s.
func entropy(s string) float64 {
size := len([]byte(s))
if size == 0 {
return 0
}
// Calculate the probabilities for a given byte in string s.
var (
l = float64(size)
m = make(map[byte]float64)
f float64
)
for i := 0; i < size; i++ {
m[s[i]]++
}
for _, c := range m {
f += c * math.Log2(c)
}
return math.Log2(l) - f/l
}
// idealEntropy calculates the ideal Shannon entropy of a string of length n.
func idealEntropy(n int) float64 {
probability := 1.0 / float64(n)
return -1.0 * float64(n) * probability * math.Log(probability) / math.Log(2.0)
}
func splitAfter(s string, separators []rune) (out []string) {
var (
j int
p = []rune(s)
l = len(p)
)
if l == 0 {
return
}
for i := 0; i < l; i++ {
if strings.ContainsRune(string(separators), p[i]) {
k := i
for k < l && strings.ContainsRune(string(separators), p[k]) {
k++
}
out = append(out, s[j:k])
i, j = k, k
}
}
if j > 0 && j < l-1 {
out = append(out, s[j:])
}
if len(out) == 0 {
out = append(out, s)
}
return
}
func isSpace(c byte) bool {
return c == ' ' || c == '\t' || c == '\n' || c == '\r'
}

+ 54
- 0
entropy_test.go View File

@ -0,0 +1,54 @@
package scrub
import "testing"
func TestEntropyScrubber(t *testing.T) {
scrubber := EntropyScrubber{
Whitespace: defaultWhitespace,
Threshold: defaultEntropyThreshold,
}
tests := []struct {
Test, Want string
}{
// No scrubbing:
{"ps axufwww", "ps axufwww"},
// Secret scrubbing:
{"chpasswd root coRrecth0rseba++ery9.23.2007staple$", "chpasswd root *redacted*"},
}
for _, test := range tests {
t.Run(test.Test, func(it *testing.T) {
if v := scrubber.Scrub(test.Test); v != test.Want {
t.Errorf("expected %q to return %q, got %q", test.Test, test.Want, v)
}
})
}
}
func TestEntropyShannon(t *testing.T) {
var tests = []struct {
Test string
Want float64
}{
{"", 0},
{"1223334444", 1.8464393446710152},
{"password123", 3.2776134368191157},
{"Tr0ub4dour&3", 3.2516291673878226},
{"correcthorsebatterystaple", 3.363856189774724},
{"coRrecth0rseba++ery9.23.2007staple$", 4.229003731107054},
{"rWibMFACxAUGZmxhVncy", 4.1219280948873624},
{"Ba9ZyWABu99[BK#6MBgbH88Tofv)vs$", 4.413716068381602},
}
for _, test := range tests {
t.Run(test.Test, func(it *testing.T) {
if v := entropy(test.Test); !almostEqual(v, test.Want) {
it.Errorf("expected %q to return %g, got %g", test.Test, test.Want, v)
}
})
}
}
const epsilon = 0.00000001
func almostEqual(a, b float64) bool {
return (a-b) < epsilon && (b-a) < epsilon
}

+ 3
- 0
go.mod View File

@ -0,0 +1,3 @@
module maze.io/x/scrub
go 1.13

+ 14
- 0
password.go View File

@ -0,0 +1,14 @@
package scrub
import (
"regexp"
)
const (
reCryptHash = `(?s)\$[-0-9a-z]+\$((?:rounds=\d+\$)?(?:` + reBase64 + `+)+\$?)`
)
var (
// CryptHash can scrub hashes in common crypt formats, such as Apache and IANA crypt hashes.
CryptHash Scrubber = regexpScrubber{regexp.MustCompile(reCryptHash), false}
)

+ 24
- 0
password_test.go View File

@ -0,0 +1,24 @@
package scrub
import "testing"
func TestCryptHash(t *testing.T) {
var tests = []struct {
Test string
Want string
}{
{"$0$testing", "$0$*redacted*"},
{`$2a$06$/3OeRpbOf8/l6nPPRdZPp.nRiyYqPobEZGdNRBWihQhiFDh1ws1tu`, `$2a$*redacted*`},
{`$6$Np2eF8019ITolL$Q4ZB.EYJdr8nD8OyNPnOTuntLbZXl3YN5r49qtRZDd9JOR.5j1s6zQ7zPekxpVi1WEQ7pYB0AJkHU61Th6Ndf0`, `$6$*redacted*`},
{`$5$rounds=80000$wnsT7Yr92oJoP28r$cKhJImk5mfuSKV9b3mumNzlbstFUplKtQXXMo4G6Ep5`, `$5$*redacted*`},
{`$sha1$c6218$161d1ac8ab38979c5a31cbaba4a67378e7e60845`, `$sha1$*redacted*`},
}
for _, test := range tests {
t.Run(test.Want, func(t *testing.T) {
if scrubbed := CryptHash.Scrub(test.Test); scrubbed != test.Want {
t.Errorf("expected %q, got %q", test.Want, scrubbed)
}
})
}
}

+ 22
- 0
pem.go View File

@ -0,0 +1,22 @@
package scrub
import "regexp"
func init() {
All = append(All, PEMDHParameters, PEMPrivateKey)
}
type pemBlockScrubber struct {
ScrubHeaders bool
}
var (
PEMDHParameters = regexpScrubber{
pattern: regexp.MustCompile(`(?s)-----BEGIN DH PARAMETERS-----\n([+0-9a-zA-Z/\n]+)\n-----END DH PARAMETERS`),
equalLength: true,
}
PEMPrivateKey = regexpScrubber{
pattern: regexp.MustCompile(`(?s)-----BEGIN (?:[A-Z]+ )PRIVATE KEY-----\n([+0-9a-zA-Z/\n]+)\n-----END (?:[A-Z]+ )PRIVATE KEY`),
equalLength: true,
}
)

+ 103
- 0
pem_test.go View File

@ -0,0 +1,103 @@
package scrub
import "testing"
func TestPEMDHParameters(t *testing.T) {
testScrubber(t, PEMDHParameters, []testScrubberCase{
{Test: testDHParams, Want: wantDHParams},
})
}
func TestPEMPrivateKey(t *testing.T) {
testScrubber(t, PEMPrivateKey, []testScrubberCase{
{Test: testRSAPrivateKey, Want: wantRSAPrivateKey},
{Test: testSSHEd25519Key, Want: wantSSHEd25519Key},
})
}
var (
testDHParams = `-----BEGIN DH PARAMETERS-----
MIGHAoGBANob7KUDV/hqwO2TRlE119oYOxrStVPMaqzuPCnXBX0dw5Yn9lYjoMI9
6jSberXoK58qVZzeCIfgcUg7+JeTuEvD2zBI9iayb6NlYV6/6z6vKQHmCeGmK1k/
g1c4LRyWfXDsN94o5oYO7LfyIdZCoOG+mkqO60glbQGBNKMXvPcLAgEC
-----END DH PARAMETERS-----
`
wantDHParams = `-----BEGIN DH PARAMETERS-----
*redacted*******************************************************
*redacted*******************************************************
*redacted***********************************************
-----END DH PARAMETERS-----
`
testRSAPrivateKey = `-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEAtsR8bBSc4ippQhkAO2iPdpQwxjdVfvmHgISQ6pj9lK+XPOu2
qLknjMd+WxN04BSLKup+Pu+3hbnhiyFG9wbipXQxbhLKFSelDfeKpYEQApTbF0Ec
wQ0etVnc0V4mElmmOVN8RIG8HfjXB3NO9lInDvdXoc5IMGEzP9i4wqfGpfxKUkAu
3Mg8/phziJNFJ7f59FlFJrcjOsfmPCV/KqO4zlGY0tLEd0XJ03+0BY6Vs+D3KyyU
6OZIWjKHYM5MNnDTdSdEfK21e/ul+EYD77bsTLmbHDN+HU0lgNzWJTigpsZymEtp
8G3GUsJMBxh8zT/uLIT7vfuuUqnO4fhZxiW0XwIDAQABAoIBACW+QeuX/iX+mCoc
O54JI8dbJw9oEfHc2gzCU6L+4S85a8Qa8We6hN5fvEpWpEY6N9su/c9FdeLZ1igD
QUJ2W4vLiQGwQ6dGvqE5w5oWIxZFY1FUEvoTGYpd+moKRVZ4yQkBoqILIKwX3WAe
geoAYSyIC8LQdLv49rpyqQUZ3L4eGDKS8Y9mFHOsYtuKBJo1EgPg4+2gJxntZMLH
DrH3ZUYB1UZlggdHyoi77oF0wCvwacdYtRDD5K0opGUduyL5+IkWGklC03Gt1dwC
7RU/zeaGYbI1nybBgeZvKzSAX2S52ImKQSMFWiLsP1DF7yghN1105rVEo+RzPkJe
ccgWoUECgYEA6DFHlXXXHTBx2MtclhfTMCq9uICjsOrxG2cyKFJV233ETiISSAum
A4kIC01stk8ARVWXYBWGyguBEkHA3Y42gCCZoU3SZNZuM0RScfR3iJneCztaCrq4
5ntvqEfOkRHk8/5tFQ/KH4i4LCOy40w6xQ2tjHJLdjBNDafd/t9J7C8CgYEAyYHf
IvxXiN2bkRf9mD72Epf06LgBk6Of4xP61vU+Ld4OMcF1TviL9G8xMeeftFRpCqpL
njGqrGcuuSU3a2PsLTb+1q69t9bSWK2oHX9fj8PgSWN91kRqZKA/QnN1aXv2/ww3
LtVk8bn9+fZwnBNEMi/l67erWuFFqihY2TVPvtECgYAbwkB3mtXz1GXX8EAKZaDG
4mU6GI75SK04hHbXoThIfFmqqaIb4OChDZHboA7+IKW8pEXro8cwgn2UzC2djzHu
0XbsdNxRV91m3aUpoHtl5ldIankSTU5rp5gquyLz7vq7PNCXswKMEJFMHZx2Vhe0
lTUJVGS3JYEgv8/nd5Rj2wKBgExuYE6K3EDjnZApQ10t9HQVAyYKNT7kv06IU4qZ
Nt567XNd57rud1ddnZFKQ79IjRcohMoaGJyP/p7nSOAI5Jo50+tmGDvU1bAhHjUi
DQMgzr/HZwGQrbJBPf1cgdpi1MrkvUGcW098tqLLIOdyP1mx5UnFPs+Xxq7F4v1w
RTEhAoGBAI3rGEqN5f1N+k4EQf1Op2M+//hhwHpyoryc9W8ed/BF3i22IPQeRjlI
g9bxjODr2RRt/GOH9WlNd5OkcMkqQfRQ75NNEFNW5I85oWP3yuRCPbav51BB1DrF
Rlg5XnHMNc5SXbMjTQpUjjNVefoNss6FXYZTEvR8k/0CBjh9MFQH
-----END RSA PRIVATE KEY-----
`
wantRSAPrivateKey = `-----BEGIN RSA PRIVATE KEY-----
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************************
*redacted*******************************************
-----END RSA PRIVATE KEY-----
`
testSSHEd25519Key = `-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACB4nWmwuPMYbVcNVzWhWGsu1CRJ0N9Jq+tDXvP0yG/u/AAAAIiRtuIakbbi
GgAAAAtzc2gtZWQyNTUxOQAAACB4nWmwuPMYbVcNVzWhWGsu1CRJ0N9Jq+tDXvP0yG/u/A
AAAECquNjGZMIC/5JAJ3qFpcmmJ439t5iD3XWsK/dXUjP8RHidabC48xhtVw1XNaFYay7U
JEnQ30mr60Ne8/TIb+78AAAAAAECAwQF
-----END OPENSSH PRIVATE KEY-----
`
wantSSHEd25519Key = `-----BEGIN OPENSSH PRIVATE KEY-----
*redacted*************************************************************
*redacted*************************************************************
*redacted*************************************************************
*redacted*************************************************************
*redacted***********************
-----END OPENSSH PRIVATE KEY-----
`
)

+ 122
- 0
scrub.go View File

@ -0,0 +1,122 @@
// Package scrub offers data scrubbing options for protecting sensitive data.
package scrub
import (
"io"
"log"
"regexp"
"strings"
)
var (
// Replacement string.
Replacement = `*redacted*`
// ReplaceChar is used for equal length replacement.
ReplaceChar = '*'
)
// Common patterns
const (
reBase64 = `[0-9A-Za-z./+=,$-]`
)
// Scrubber redacts sensitive data.
type Scrubber interface {
// Scrub a string.
Scrub(string) string
}
// Scrubbers are zero or more Scrubber that act as a single scrubber.
type Scrubbers []Scrubber
// Scrub with all scrubbers, the first scrubber to alter the input will return the scrubbed output.
func (scrubbers Scrubbers) Scrub(s string) string {
for _, scrubber := range scrubbers {
if v := scrubber.Scrub(s); v != s {
log.Printf("%T %q -> %q", scrubber, s, v)
return v
}
}
return s
}
// All registered scrubbers in safe evaluation order.
var All = Scrubbers{
CommandScrubber,
CryptHash,
PEMDHParameters,
PEMPrivateKey,
EntropyScrubber{
Whitespace: defaultWhitespace,
Threshold: defaultEntropyThreshold,
},
}
// WrapReader acts as an io.Reader.
func WrapReader(r io.Reader, scrubbers Scrubbers) io.Reader {
if len(scrubbers) == 0 {
return r
}
return reader{r, scrubbers}
}
// re is shorthand to compile a regular expression
func re(pattern string) *regexp.Regexp {
return regexp.MustCompile(pattern)
}
// ScrubEqualLength redacts match from in, keeping the same length.
func scrubEqualLength(in, match string) string {
var (
l = len(match)
replace = []byte(strings.Repeat(string(ReplaceChar), l))
)
copy(replace, Replacement)
// We also keep newline characters from the original string.
for i, c := range []byte(match) {
if c == '\n' || c == '\r' || c == '\t' {
replace[i] = c
if i+1 < l {
copy(replace[i+1:], Replacement)
}
}
}
// Replace output.
return strings.Replace(in, match, string(replace), 1)
}
type reader struct {
r io.Reader
s Scrubbers
}
func (r reader) Read(p []byte) (n int, err error) {
if n, err = r.r.Read(p); n > 0 {
for _, s := range r.s {
copy(p[:n], s.Scrub(string(p[:n])))
}
}
return
}
type regexpScrubber struct {
pattern *regexp.Regexp
equalLength bool
}
func (r regexpScrubber) Scrub(s string) string {
matches := r.pattern.FindStringSubmatch(s)
if len(matches) > 0 {
for _, match := range matches[1:] {
if r.equalLength {
s = scrubEqualLength(s, match)
} else {
s = strings.Replace(s, match, Replacement, 1)
}
}
}
return s
}

+ 49
- 0
scrub_test.go View File

@ -0,0 +1,49 @@
package scrub
import (
"strings"
"testing"
)
type testScrubberCase struct {
Test, Want string
}
func testScrubber(t *testing.T, s Scrubber, tests []testScrubberCase) {
t.Helper()
for _, test := range tests {
t.Run(strings.SplitN(test.Want, "\n", 2)[0], func(it *testing.T) {
if result := s.Scrub(test.Test); result != test.Want {
it.Fatalf("expected %q, got %q", test.Want, result)
}
})
}
}
func TestAll(t *testing.T) {
testScrubber(t, All, []testScrubberCase{
// Command tests.
{"/usr/bin/mysqldump", "/usr/bin/mysqldump"},
{"mysqldump -u john -h localhost", "mysqldump -u john -h localhost"},
{"mysqldump -u john -p testing -h localhost", "mysqldump -u john -p*redacted* -h localhost"},
{"/opt/sap/bin/mysqldump -u john -p testing -h localhost", "/opt/sap/bin/mysqldump -u john -p*redacted* -h localhost"},
// Entropy tests.
// No scrubbing:
{"ps axufwww", "ps axufwww"},
// Secret scrubbing:
{"chpasswd root coRrecth0rseba++ery9.23.2007staple$", "chpasswd root *redacted*"},
// Password tests.
{"$0$testing", "$0$*redacted*"},
{`$2a$06$/3OeRpbOf8/l6nPPRdZPp.nRiyYqPobEZGdNRBWihQhiFDh1ws1tu`, `$2a$*redacted*`},
{`$6$Np2eF8019ITolL$Q4ZB.EYJdr8nD8OyNPnOTuntLbZXl3YN5r49qtRZDd9JOR.5j1s6zQ7zPekxpVi1WEQ7pYB0AJkHU61Th6Ndf0`, `$6$*redacted*`},
{`$5$rounds=80000$wnsT7Yr92oJoP28r$cKhJImk5mfuSKV9b3mumNzlbstFUplKtQXXMo4G6Ep5`, `$5$*redacted*`},
{`$sha1$c6218$161d1ac8ab38979c5a31cbaba4a67378e7e60845`, `$sha1$*redacted*`},
// PEM tests.
{Test: testDHParams, Want: wantDHParams},
{Test: testRSAPrivateKey, Want: wantRSAPrivateKey},
{Test: testSSHEd25519Key, Want: wantSSHEd25519Key},
})
}

Loading…
Cancel
Save