-
Notifications
You must be signed in to change notification settings - Fork 25
/
Copy pathpatterns.yml
236 lines (221 loc) · 12 KB
/
patterns.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
name: Generic Secrets / Passwords
patterns:
- name: Generic Passwords
type: generic_passwords
experimental: true
regex:
version: 0.5
pattern: |
[a-zA-Z0-9!.,$%&*+?^_`{|}()[\]\\/~-][a-zA-Z0-9\t !.,$%&*+?^_`{|}()[\]\\/~-]*
start: |
(?:\A|[^a-zA-Z0-9])(?i)[a-z0-9_.-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
end: |
(\z|[\r\n'"])
additional_not_match:
# something that means "password" or a placeholder name - either a variable or a placeholder
# a literal value or config switch
- ^(?i)_?\)?((a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|key|tok|pw)|redacted|placeholder|dummy|thephrase|write|read|on|off|true|false|none|value|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n|f[0-9]{1,2}|[a-zA-Z]),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
# Python type hints, Swift typing
- ^\s*(?:(?:typing\.)?(?:(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*|(?:int|str|float|(?:typing.)?Any|None|bytes|bool|ReadableBuffer)\s*(?:[,|].*)?|(?:Int|Swift\.Int|Int32)\.*))\s*$
# ..., \, , \n, \0, ',' and other single chars, smilies, hex, digits, nothing at all,
# directories, regex, format string placeholder, urllib demo passphrase, "optional" in docs, a variable substitution, or surrounded by brackets of various kinds
# all with possible ',' and surrounding whitespace, possibly with a following comment
- ^\s*(?:\.\.\.|\\|\\n|\\0|\?|\$\(|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.|(/[a-zA-Z0-9./_-]+/)?[a-zA-Z0-9]+(\.(pem|crt|key|cer|pub|der)|_rsa))|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
# function definitions, e.g. Javascript, function calls or variable declaration
- ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$|\{\{[^}]+\}\}|\$\{\{|\{\}$|\[\]$|(0x)?%[0-9]+x|%[dusx]\.$
# reference to a member variable, index into a variable, bash variables, perl hash key index, environment vars
- ^\s*(?:(?:self|this)\.[a-zA-Z_][a-zA-Z0-9_.]+[,[]?|[a-zA-Z0-9_.]+\[(?:[a-zA-Z0-9_.]+)?\]?|\$(?:[1-9]|[A-Za-z0-9_]+)\{?|os\.environ\[[^\]]\]|process\.env\.[A-Z0-9_]+)\s*(?:,|\|\||&&)?\s*$|`(%s|\+)
test:
data: password=Password123
start_offset: 9
end_offset: -1
expected:
- name: passwords.js
start_offset: 14
end_offset: 25
- name: passwords.js
start_offset: 41
end_offset: 62
- name: passwords.js
start_offset: 97
end_offset: 112
comments:
- "Likely to cause large numbers of false positives - use with caution"
- "`password`, `secret`, `key`, or password like prefix (fuzzy)"
- "Delimiters like `=` or `:` (with padding)"
- "String with a number of chars until a breaking char"
- "Not matching variables, placeholders or common configuration constants such as 'read' and 'write'"
- name: Generic Passwords (fewer FPs)
type: generic_passwords_fewer_fps
regex:
version: 0.1
pattern: |
((?i)[a-z0-9_.-]*(api|auth[a-z]*|jwt|mysql|db)[_.-]?)?((?i)pass?(wo?r?d|code|phrase)|secret|token|key)([_-][A-Za-z0-9]+){0,4}_{0,2}(["'`]|[ \t]+As[ \t]+String)?[\t ]*(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)[\t ]*([br]?"[A-Za-z0-9_/+.!-]+"|[br]?'[A-Za-z0-9_/+.!-]+')
start: |
\A|[^0-9A-Za-z]
end: |
\z|[^A-Za-z0-9]
additional_not_match:
- |-
^[A-Za-z0-9_.-]*(key|KEY|[Tt]oken|TOKEN)(_[a-zA-Z]+)?['"]?\s*([:=]|=>)\s*["']([Ee]mploye[er]|[Ss]taff|([Ss]earch)?[Rr]esult|[a-z][a-zA-Z]+CSX[A-Z][A-Za-z]+|[A-Za-z]*[Bb]ase(64|32|58)|object|claret|assigns?|clean|contains|error|expand|generate|hoist|indent(ation)?|invert|jumps?|pairs?|param(eter)?s?|pop|rewrite|temp(orary)?|token(s|i[sz]e)?|type|((un)?(quote|shift|wrap|finished))|[a-z]{2,10}([A-Z][a-z]{1,15}){1,6}|(compile|is|has|make|add|each|check|close|cache|format|tag|get|set)([A-Z][A-Za-z]+)?|gadget|classic|(try_)?(base|mode|grade|model)|words|identifier|[a-z.-]+\.(jpe?g|(x|ht)ml|txt|docx?|xlsx?|pdf|png)|enabled|name\.invalidPattern|\.|\.data-api|expect|file|config|ansi|Default(Type)?|Cache-Control|((notD|d)eepE|e)qual|name|NAME|package|version|VERSION|start|end|step|async|Event|throws|ok|notOK|verbose|push(Result)?|slimAssertions|(p|notP)ropEqual|((notS|s)trict|not)Equal|value|prev|next|year|key[0-9]?|destroy|[a-z]+EventListeners|timeout|str(ing)?|hmac|uuid|update|find|true|false|val|VAL|REDACTED|redacted|nop|F[0-9]{1,2}|[A-Za-z0-9]|[Nn][ui]ll|[Nn]one|[a-z_]+\.((tf)?state|id|key)|(hibernate|ws|err|i18n|employee|bs|org|com|sun|java)(\.[a-zA-Z0-9_]+){1,4}|[A-Z_]+_KEY)["']$
- |-
(?i)(token|key)[_-](name|format|type|enabled|success|type|method)\b
- |-
^(?i)token(_[A-Z]+)?['"]?\s*[:=]\s*['"](barline|parenthesis|qualified|suport|symbol|statementEnd|singleLineTitle|character|pageBreak|operator|optionalTitle|option|zupfnoter|chordname|macro|error|escape|indent|term|titleUnderline|tag|link|literal|(other|table)Block|list|value|control|set|support|injections|array|doc|source|heading|tokens|storage|empty|newline|empty_line|keyword|(line)?comment|meta|[lr]?paren|class|punctuation|regexp?|constant|string|entity|invalid|support|variable|multiline|language|paren|markup|singleline|nospell|text|array|doc|source|heading|tokens)(\.{1,2}[A-Za-z0-9_-]+){0,6}[!.]?["']$
- "^KEY_[A-Z]+[0-9]{0,3}: 'k[a-zA-Z0-9]{1,6}'$"
- |-
['"` ](/dev/u?random|(/[a-zA-Z0-9./_-]+/)?[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)|https?://.*|file://.*)['"`]$
test:
data: password="Password123"
start_offset: 0
end_offset: 22
comments:
- "Expect many false positives if run against vendored-in code, such as JS libraries - use with caution"
- "`password`, `secret`, `key`, `token` etc. password-like prefix (fuzzy)"
- "Delimiters like `=` or `:` (with padding)"
- "Matches fewer characters (A-Za-z0-9_/+.!-), and requires matching quotes around the string"
- "Attempts to remove variables, placeholders, and common configuration constants such as 'read' and 'write'"
- name: Generic Password with hex encoded secrets
type: generic_passwords_hex
regex:
version: 0.1
pattern: |
[0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64}
start: |
(?:\A|[^a-zA-Z0-9])(?i)[a-z0-9._-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
end: |
(\z|[\r\n'"])
test:
data: password=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
start_offset: 9
end_offset: -1
comments:
- "`password`, `secret`, `key`, or password like prefix (fuzzy)"
- "Delimiters like `=` or `:` (with padding)"
- "Has to be a token-like value - a 32, 40 or 64 character hex string"
- name: Generic Password with Base64 encoded secrets
type: generic_passwords_base64
regex:
version: 0.1
pattern: |
(([A-Za-z0-9+/]){4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)
start: |
(?:\A|[^a-zA-Z0-9])(?i)[a-z0-9._-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
end: |
(\z|[\r\n'"])
additional_match:
- '[0-9]'
- '[A-Z]'
- '[a-z]'
- '^.{12,}$'
test:
data: password="AAAAAAAAAAAa00=="
start_offset: 10
end_offset: 26
comments:
- "The Base64 must contain numbers, upper case and lower case and be at least 12 characters long"
- "`password`, `secret`, `key`, or password like prefix (fuzzy)"
- "Delimiters like `=` or `:` (with padding)"
- name: Generic Password with URI-safe Base64 encoded secrets
type: generic_passwords_base64_uri
regex:
version: 0.1
pattern: |
(([A-Za-z0-9_-]){4})*([A-Za-z0-9_-]{4}|[A-Za-z0-9_-]{3}=|[A-Za-z0-9_-]{2}==)
start: |
(?:\A|[^a-zA-Z0-9])(?i)[a-z0-9._-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
end: |
(\z|[\r\n'"])
additional_match:
- '[0-9]'
- '[A-Z]'
- '[a-z]'
- '^.{12,}$'
test:
data: password="AAAAAAAAAAAa00=="
start_offset: 10
end_offset: 26
comments:
- "The Base64 must contain numbers, upper case and lower case and be at least 12 characters long"
- "`password`, `secret`, `key`, or password like prefix (fuzzy)"
- "Delimiters like `=` or `:` (with padding)"
- "This matches _- instead of +/, for URI-safe Base64"
- name: UUIDs
type: uuids
regex:
version: 0.1
pattern: |
(?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
start: |
\A|[^0-9A-Fa-f-]
end: |
\z|[^0-9A-Fa-f-]
additional_not_match:
- ^12345678-1234-5678-1234-567812345678$
- ^00000000-0000-0000-0000-000000000000$
- ^(?i)00010203-0405-0607-0809-0a0b0c0d0e0f$
- ^(?i)12345678-1234-1234-1234-123456789abc$
test:
- data: 10203040-5060-7080-90a0-b0c0d0e0f000
start_offset: 0
end_offset: -1
expected:
- name: uuids.txt
start_offset: 12
end_offset: 48
- name: uuids.txt
start_offset: 49
end_offset: 85
- name: uuids.txt
start_offset: 131
end_offset: 167
- name: Bearer Tokens
type: bearer_tokens
regex:
version: 0.2
pattern: |
[a-zA-Z0-9_.=/+:-]{12,}
start: |
(Authorization: |['"])([Bb]earer |[Tt]oken (token=)?)
end: |
\z|[\s'"]
additional_not_match:
- ^(?:letmein|Oracle|SuperSecretString|foo|ababbdbbebbbebdbbe5538003023|XYZ_INVALID_ACCESTOKEN_XYZ|QQ==|Shizuku|mF_9.B5f-4.1JqM|h480djs93hd8|SlAV32hkKG|YmVlcDpib29w)$
- ^(?i)(?:dummy|fake|bearer|auth|invalid|your|my|the|undefined|github|oidc|database)(?:_api)?(?:_?token|key|secret)?$
- ^(?i)(?:[a-z0-9]|XYZ|ABC|123|.*_token)$
- (^(?i)(x+|y+|z+|a+|\.+|.*\.\.\.)$|(?i)x{5})
expected:
- name: bearer.txt
start_offset: 45
end_offset: 64
- name: bearer.txt
start_offset: 99
end_offset: 118
test:
- data: "Authorization: Bearer FAKETOKENFORTESTING"
start_offset: 23
end_offset: 42
comments:
- As used in an Authorization header
- We try to remove common placeholders
- Lower length limit of 12 to remove common false positives on "Token <common word>", since most words are below 12 characters in length
- name: OAuth client secret and ID pair
type: oauth_client_secret
regex:
version: 0.1
pattern: |
(?i)client[_.-]?Id\s*([:=]|[=-]>|to|[!=]={1,2}|<>)\s*['"`]?[^\s'"`[\]{}()<>]+['"`]?\s*[,\r\n]\s*\bclient[_.-]?Secret\s*([:=]|[=-]>|to|[!=]={1,2}|<>)\s*['"`]?[^\s'"`[\]{}()<>]+['"`]?
start: |
\A|\b
end: |
\z|\b
additional_not_match:
- ^(?i)client[_.-]?id\s*[:=]?\s*(string|str|None)\b|\.\.\.|\(string\)|(?i)Client(ID|Secret)[a-z]|^(?i)client[_.-]?id["'`]\)|"\$\{
- ^(?i)client[_.-]?id\s*[=:]\s*([a-z.]+(\[|\.get\())?["'`]?(\$\{|@)?[a-z0-9_.-]*((client|app)[_.-]?id|key)\b
- (?i)client[_.-]?secret\s*[=:]\s*["'`]?(\$\{|@)?[a-z_.-]*(secret|token)\b
- ^(?i)client[_.-]?Id(:.,|:\s*client[_.-]?secret:)
- xxxxx|\?\?\?\?\?|example|00000|123-?45|['"][^'"\s]{1,5}['"]|(?i)<client[_-]?id>
test:
- data: |
client_id = '1234567890'
client_secret = 'thisisaverysecretkey'