Browse Source

Refactoring; Dropped Reader because it does not work.

master
maze 1 year ago
parent
commit
0896af8837
7 changed files with 121 additions and 60 deletions
  1. +1
    -1
      command.go
  2. +36
    -19
      entropy.go
  3. +24
    -2
      entropy_test.go
  4. +4
    -1
      password.go
  5. +3
    -8
      pem.go
  6. +9
    -4
      scrub.go
  7. +44
    -25
      scrub_test.go

+ 1
- 1
command.go View File

@ -16,7 +16,7 @@ var Command = CommandScrubber{
type CommandScrubber map[string][]*regexp.Regexp
func (cs CommandScrubber) Scrub(s string) string {
f := splitAfter(s, []rune(defaultWhitespace))
f := splitAfter(s, []rune(DefaultWhitespace))
for i, p := range f {
base := filepath.Base(strings.TrimSpace(p))
if args, ok := cs[base]; ok {


+ 36
- 19
entropy.go View File

@ -5,29 +5,47 @@ import (
"strings"
)
// Defaults for the Entropy scrubber.
const (
defaultWhitespace = " \t\r\n"
// DefaultWhitespace is used to split the given input string into tokens that get scrubbed individually.
DefaultWhitespace = " \t\r\n="
// defaultEntropyThreshold is chosen to not match most UNIX shell commands, but it does match
// DefaultEntropyThreshold is chosen to not match most UNIX shell commands, but it does match
// passwords with sufficient complexity; use with care!
defaultEntropyThreshold = 3.75
DefaultEntropyThreshold = 3.75
// IdealEntropyCorrection is used to calculate how much of ideal entropy should be in the string to be considered for scrubbing.
IdealEntropyCorrection = 0.75
)
// EntropyScrubber splits the input by Whitespace and matches each part's entropy with the configured threshold.
type EntropyScrubber struct {
// Whitespace characters.
Whitespace string
Threshold float64
// Threshold for scrubbing. If not set the ideal entropy is calculated based on the length of the input string.
Threshold float64
// Correction threshold matching based on the ideal entropy if the Threshold is empty. If not set, the
// correction is set to the IdealEntropyCorrection.
Correction float64
}
// Scrub string s by first splitting the input based on Whitespace and then analyzing the entropy of each field.
func (es EntropyScrubber) Scrub(s string) string {
var (
whitespace = es.Whitespace
threshold = es.Threshold
correction = es.Correction
)
if whitespace == "" {
whitespace = defaultWhitespace
whitespace = DefaultWhitespace
}
if threshold == 0 {
threshold = defaultEntropyThreshold
if correction == 0 {
correction = IdealEntropyCorrection
}
threshold = idealEntropy(len(s)) * correction
}
f := splitAfter(s, []rune(whitespace))
@ -39,16 +57,25 @@ func (es EntropyScrubber) Scrub(s string) string {
return strings.Join(f, "")
}
// Entropy scrubs all high-entropy strings from s.
// Entropy scrubs all high-entropy strings from s based on the ideal entropy for a string of len(s).
func Entropy(s string) string {
return EntropyWithThreshold(s, defaultEntropyThreshold)
return EntropyWithThreshold(s, idealEntropy(len(s))*IdealEntropyCorrection)
}
// idealEntropy calculates the ideal Shannon entropy of a string of length n.
func idealEntropy(n int) float64 {
if n == 0 {
return 0
}
probability := 1.0 / float64(n)
return -1.0 * float64(n) * probability * math.Log(probability) / math.Log(2.0)
}
// EntropyWithThreshold is like Entropy with a custom threshold.
func EntropyWithThreshold(s string, threshold float64) string {
f := strings.Fields(s)
for i, p := range f {
if entropy(p) > threshold {
if entropy(strings.TrimSpace(p)) >= threshold {
f[i] = Replacement
}
}
@ -77,12 +104,6 @@ func entropy(s string) float64 {
return math.Log2(l) - f/l
}
// idealEntropy calculates the ideal Shannon entropy of a string of length n.
func idealEntropy(n int) float64 {
probability := 1.0 / float64(n)
return -1.0 * float64(n) * probability * math.Log(probability) / math.Log(2.0)
}
func splitAfter(s string, separators []rune) (out []string) {
var (
j int
@ -110,7 +131,3 @@ func splitAfter(s string, separators []rune) (out []string) {
}
return
}
func isSpace(c byte) bool {
return c == ' ' || c == '\t' || c == '\n' || c == '\r'
}

+ 24
- 2
entropy_test.go View File

@ -2,10 +2,32 @@ package scrub
import "testing"
func TestEntropy(t *testing.T) {
tests := []struct {
Test, Want string
}{
{"", ""},
{"1223334444", "1223334444"},
{"password123", Replacement},
{"Tr0ub4dour&3", Replacement},
{"test correcthorsebatterystaple", "test correcthorsebatterystaple"},
{"test coRrecth0rseba++ery9.23.2007staple$", "test " + Replacement},
{"rWibMFACxAUGZmxhVncy test", Replacement + " test"},
{"test Ba9ZyWABu99[BK#6MBgbH88Tofv)vs$ test", "test " + Replacement + " test"},
}
for _, test := range tests {
t.Run(test.Test, func(it *testing.T) {
if v := Entropy(test.Test); v != test.Want {
it.Errorf("expected %q to return %q, got %q", test.Test, test.Want, v)
}
})
}
}
func TestEntropyScrubber(t *testing.T) {
scrubber := EntropyScrubber{
Whitespace: defaultWhitespace,
Threshold: defaultEntropyThreshold,
Whitespace: DefaultWhitespace,
Threshold: DefaultEntropyThreshold,
}
tests := []struct {
Test, Want string


+ 4
- 1
password.go View File

@ -10,5 +10,8 @@ const (
var (
// CryptHash can scrub hashes in common crypt formats, such as Apache and IANA crypt hashes.
CryptHash Scrubber = regexpScrubber{regexp.MustCompile(reCryptHash), false}
CryptHash Scrubber = regexpScrubber{
pattern: regexp.MustCompile(reCryptHash),
equalLength: false,
}
)

+ 3
- 8
pem.go View File

@ -2,19 +2,14 @@ package scrub
import "regexp"
func init() {
All = append(All, PEMDHParameters, PEMPrivateKey)
}
type pemBlockScrubber struct {
ScrubHeaders bool
}
var (
// PEMDHParameters scrubs a PEM DH PARAMETERS block retaining its original length.
PEMDHParameters = regexpScrubber{
pattern: regexp.MustCompile(`(?s)-----BEGIN DH PARAMETERS-----\n([+0-9a-zA-Z/\n]+)\n-----END DH PARAMETERS`),
equalLength: true,
}
// PEMPrivateKey scrubs a PEM PRIVATE KEY block retaining its original length.
PEMPrivateKey = regexpScrubber{
pattern: regexp.MustCompile(`(?s)-----BEGIN (?:[A-Z]+ )PRIVATE KEY-----\n([+0-9a-zA-Z/\n]+)\n-----END (?:[A-Z]+ )PRIVATE KEY`),
equalLength: true,


+ 9
- 4
scrub.go View File

@ -2,7 +2,6 @@
package scrub
import (
"io"
"regexp"
"strings"
)
@ -46,8 +45,8 @@ var All = Scrubbers{
PEMDHParameters,
PEMPrivateKey,
EntropyScrubber{
Whitespace: defaultWhitespace,
Threshold: defaultEntropyThreshold,
Whitespace: DefaultWhitespace,
Threshold: DefaultEntropyThreshold,
},
}
@ -78,6 +77,7 @@ func scrubEqualLength(in, match string) string {
return strings.Replace(in, match, string(replace), 1)
}
/*
// Reader acts as an io.Reader.
func Reader(r io.Reader, scrubbers Scrubbers) io.Reader {
if len(scrubbers) == 0 {
@ -94,13 +94,18 @@ type reader struct {
// Read from the underlying Reader, then scrub the output buffer.
func (r reader) Read(p []byte) (n int, err error) {
if n, err = r.r.Read(p); n > 0 {
log.Printf("read %d: %q", n, p[:n])
s := string(p[:n])
if v := r.s.Scrub(s); v != s {
n = copy(p, s)
m := copy(p, s)
for i := m; m < n; i++ {
p[i] = byte(ReplaceChar)
}
}
}
return
}
*/
type regexpScrubber struct {
pattern *regexp.Regexp


+ 44
- 25
scrub_test.go View File

@ -22,30 +22,49 @@ func testScrubber(t *testing.T, s Scrubber, tests []testScrubberCase) {
}
}
var testAllCases = []testScrubberCase{
// Command tests.
{"/usr/bin/mysqldump", "/usr/bin/mysqldump"},
{"mysqldump -u john -h localhost", "mysqldump -u john -h localhost"},
{"mysqldump -u john -p testing -h localhost", "mysqldump -u john -p*redacted* -h localhost"},
{"/opt/sap/bin/mysqldump -u john -p testing -h localhost", "/opt/sap/bin/mysqldump -u john -p*redacted* -h localhost"},
// Entropy tests.
// No scrubbing:
{"ps axufwww", "ps axufwww"},
// Secret scrubbing:
{"chpasswd root coRrecth0rseba++ery9.23.2007staple$", "chpasswd root *redacted*"},
// Password tests.
{"$0$testing", "$0$*redacted*"},
{`$2a$06$/3OeRpbOf8/l6nPPRdZPp.nRiyYqPobEZGdNRBWihQhiFDh1ws1tu`, `$2a$*redacted*`},
{`$6$Np2eF8019ITolL$Q4ZB.EYJdr8nD8OyNPnOTuntLbZXl3YN5r49qtRZDd9JOR.5j1s6zQ7zPekxpVi1WEQ7pYB0AJkHU61Th6Ndf0`, `$6$*redacted*`},
{`$5$rounds=80000$wnsT7Yr92oJoP28r$cKhJImk5mfuSKV9b3mumNzlbstFUplKtQXXMo4G6Ep5`, `$5$*redacted*`},
{`$sha1$c6218$161d1ac8ab38979c5a31cbaba4a67378e7e60845`, `$sha1$*redacted*`},
// PEM tests.
{Test: testDHParams, Want: wantDHParams},
{Test: testRSAPrivateKey, Want: wantRSAPrivateKey},
{Test: testSSHEd25519Key, Want: wantSSHEd25519Key},
}
func TestAll(t *testing.T) {
testScrubber(t, All, []testScrubberCase{
// Command tests.
{"/usr/bin/mysqldump", "/usr/bin/mysqldump"},
{"mysqldump -u john -h localhost", "mysqldump -u john -h localhost"},
{"mysqldump -u john -p testing -h localhost", "mysqldump -u john -p*redacted* -h localhost"},
{"/opt/sap/bin/mysqldump -u john -p testing -h localhost", "/opt/sap/bin/mysqldump -u john -p*redacted* -h localhost"},
// Entropy tests.
// No scrubbing:
{"ps axufwww", "ps axufwww"},
// Secret scrubbing:
{"chpasswd root coRrecth0rseba++ery9.23.2007staple$", "chpasswd root *redacted*"},
// Password tests.
{"$0$testing", "$0$*redacted*"},
{`$2a$06$/3OeRpbOf8/l6nPPRdZPp.nRiyYqPobEZGdNRBWihQhiFDh1ws1tu`, `$2a$*redacted*`},
{`$6$Np2eF8019ITolL$Q4ZB.EYJdr8nD8OyNPnOTuntLbZXl3YN5r49qtRZDd9JOR.5j1s6zQ7zPekxpVi1WEQ7pYB0AJkHU61Th6Ndf0`, `$6$*redacted*`},
{`$5$rounds=80000$wnsT7Yr92oJoP28r$cKhJImk5mfuSKV9b3mumNzlbstFUplKtQXXMo4G6Ep5`, `$5$*redacted*`},
{`$sha1$c6218$161d1ac8ab38979c5a31cbaba4a67378e7e60845`, `$sha1$*redacted*`},
// PEM tests.
{Test: testDHParams, Want: wantDHParams},
{Test: testRSAPrivateKey, Want: wantRSAPrivateKey},
{Test: testSSHEd25519Key, Want: wantSSHEd25519Key},
})
testScrubber(t, All, testAllCases)
}
/*
func TestReader(t *testing.T) {
for _, test := range testAllCases {
t.Run(test.Test, func(it *testing.T) {
r := Reader(bytes.NewBufferString(test.Test), All)
b, err := ioutil.ReadAll(r)
if err != nil {
it.Fatal(err)
}
if result := string(b); result != test.Want {
it.Fatalf("expected %q, got %q", test.Want, result)
}
})
}
}
*/

Loading…
Cancel
Save