Skip to content

Commit e3d8483

Browse files
Kangiehololeap
authored andcommitted
1 parent dd747b2 commit e3d8483

13 files changed

+1479
-41
lines changed

ShellCheck.cabal

+1
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ library
9797
ShellCheck.Regex
9898
other-modules:
9999
Paths_ShellCheck
100+
ShellCheck.PortageAutoInternalVariables
100101
default-language: Haskell98
101102

102103
executable shellcheck

portage/get_vars.py

+128
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#!/usr/bin/env python3
2+
# -*- coding: utf-8 -*-
3+
# Copyright 2019 The ChromiumOS Authors
4+
# All rights reserved.
5+
6+
# Redistribution and use in source and binary forms, with or without
7+
# modification, are permitted provided that the following conditions are
8+
# met:
9+
10+
# * Redistributions of source code must retain the above copyright
11+
# notice, this list of conditions and the following disclaimer.
12+
# * Redistributions in binary form must reproduce the above
13+
# copyright notice, this list of conditions and the following disclaimer
14+
# in the documentation and/or other materials provided with the
15+
# distribution.
16+
# * Neither the name of Google LLC nor the names of its
17+
# contributors may be used to endorse or promote products derived from
18+
# this software without specific prior written permission.
19+
20+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21+
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22+
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23+
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24+
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25+
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26+
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27+
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28+
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31+
# Binary License Terms
32+
33+
"""Extract eclass variable names into Haskell list format."""
34+
from __future__ import print_function
35+
import datetime
36+
import os
37+
import re
38+
import sys
39+
import textwrap
40+
# Matches a line that declares a variable in an eclass.
41+
VAR_RE = re.compile(r'@(?:ECLASS-)?VARIABLE:\s*(\w+)$')
42+
# Matches a line that declares inheritance.
43+
INHERIT_RE = re.compile(r'^[^#]*\binherit((?:\s+[\w-]+)+)$')
44+
VAR_FILE_HEADER = """module ShellCheck.PortageAutoInternalVariables (
45+
portageAutoInternalVariables
46+
) where
47+
-- This file contains the variables generated by
48+
-- portage/get_vars.py"""
49+
PORTAGE_AUTO_VAR_NAME = 'portageAutoInternalVariables'
50+
class Eclass:
51+
"""Container for eclass information"""
52+
def __init__(self, name, eclass_vars, inheritances):
53+
self.name = name
54+
self.vars = eclass_vars
55+
self.inheritances = inheritances
56+
def calculate_eclass_vars(self, eclasses):
57+
while self.inheritances:
58+
name = self.inheritances.pop()
59+
try:
60+
sub_eclass = eclasses[name]
61+
new_vars = sub_eclass.calculate_eclass_vars(eclasses).vars
62+
self.vars = self.vars.union(new_vars)
63+
except Exception:
64+
pass
65+
return self
66+
def print_var_list(eclass, eclass_vars):
67+
var_list = ' '.join(['"%s",' % v for v in sorted(eclass_vars)])
68+
print(' -- %s\n%s' %
69+
(eclass,
70+
textwrap.fill(
71+
var_list, 80, initial_indent=' ', subsequent_indent=' ')))
72+
def process_file(eclass_path):
73+
eclass_name = os.path.splitext(os.path.basename(eclass_path))[0]
74+
with open(eclass_path, 'r') as f:
75+
eclass_vars = set()
76+
eclass_inheritances = set()
77+
for line in f:
78+
line = line.strip()
79+
if not line:
80+
continue
81+
while line[-1] == '\\':
82+
line = line[:-1] + next(f).strip()
83+
match = VAR_RE.search(line)
84+
if match:
85+
var_name = match.group(1)
86+
eclass_vars.add(var_name.strip())
87+
else:
88+
match = INHERIT_RE.search(line)
89+
if match:
90+
for inheritance in re.split(r'\s+', match.group(1)):
91+
if inheritance.strip():
92+
eclass_inheritances.add(inheritance.strip())
93+
return Eclass(eclass_name, eclass_vars, eclass_inheritances)
94+
def format_eclasses_as_haskell_map(eclasses):
95+
map_entries = []
96+
join_string = '", "'
97+
for value in sorted(eclasses, key=(lambda x: x.name)):
98+
if value.vars:
99+
var_list_string = f'"{join_string.join(sorted(list(value.vars)))}"'
100+
map_entries.append(
101+
textwrap.fill(
102+
f'("{value.name}", [{var_list_string}])',
103+
80,
104+
initial_indent=' ',
105+
subsequent_indent=' '))
106+
return_string = ',\n\n'.join(map_entries)
107+
return_string = f""" Data.Map.fromList
108+
[
109+
{return_string}
110+
]"""
111+
return f"""{VAR_FILE_HEADER}\n\n
112+
-- Last Generated: {datetime.datetime.now().strftime("%x")}
113+
import qualified Data.Map
114+
{PORTAGE_AUTO_VAR_NAME} =
115+
{return_string}"""
116+
def main(argv):
117+
eclasses = {}
118+
for path in sorted(argv, key=os.path.basename):
119+
if not path.endswith('.eclass'):
120+
continue
121+
new_eclass = process_file(path)
122+
eclasses[new_eclass.name] = new_eclass
123+
eclasses_list = [
124+
value.calculate_eclass_vars(eclasses) for key, value in eclasses.items()
125+
]
126+
print(format_eclasses_as_haskell_map(eclasses_list))
127+
if __name__ == '__main__':
128+
sys.exit(main(sys.argv[1:]))

portage/get_vars_diff.py

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2020 The ChromiumOS Authors
3+
# All rights reserved.
4+
5+
# Redistribution and use in source and binary forms, with or without
6+
# modification, are permitted provided that the following conditions are
7+
# met:
8+
9+
# * Redistributions of source code must retain the above copyright
10+
# notice, this list of conditions and the following disclaimer.
11+
# * Redistributions in binary form must reproduce the above
12+
# copyright notice, this list of conditions and the following disclaimer
13+
# in the documentation and/or other materials provided with the
14+
# distribution.
15+
# * Neither the name of Google LLC nor the names of its
16+
# contributors may be used to endorse or promote products derived from
17+
# this software without specific prior written permission.
18+
19+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20+
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21+
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22+
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23+
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24+
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25+
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26+
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27+
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30+
# Binary License Terms
31+
32+
"""Generates diff of vars from get_vars.py and those existing in Data.hs."""
33+
import itertools
34+
from pathlib import Path
35+
import subprocess
36+
SCRIPT = Path(__file__).resolve()
37+
THIRD_PARTY = SCRIPT.parent.parent.parent.parent.parent
38+
# List of relative directories in which to find the eclasses.
39+
eclass_rel_dirs = (
40+
THIRD_PARTY / 'chromiumos-overlay' / 'eclass',
41+
THIRD_PARTY / 'portage-stable' / 'eclass',
42+
THIRD_PARTY / 'eclass-overlay' / 'eclass',
43+
)
44+
# Runs get_vars.py with the eclass paths and store the output.
45+
cmd = [SCRIPT.with_name('get_vars.py')] + list(
46+
itertools.chain(*(x.glob('*') for x in eclass_rel_dirs)))
47+
new_output = subprocess.check_output(cmd, encoding='utf-8').splitlines()
48+
new = []
49+
for line in new_output:
50+
if '--' in line:
51+
new.append(line.strip())
52+
elif not line.strip():
53+
continue
54+
else:
55+
new += (line.replace('"', '').replace('\n', '').split(','))
56+
# Reads the Data.hs relevant area and store the lines.
57+
data_hs = THIRD_PARTY / 'shellcheck' / 'src' / 'ShellCheck' / 'Data.hs'
58+
with data_hs.open('r', encoding='utf-8') as fp:
59+
record = False
60+
old = []
61+
for line in fp:
62+
if line.strip() == '-- autotest.eclass declared incorrectly':
63+
break
64+
if line.strip() == '-- generic ebuilds':
65+
record = True
66+
if record:
67+
if '--' in line:
68+
old.append(line.strip())
69+
elif not line.strip():
70+
continue
71+
else:
72+
old += line.replace('"', '').replace('\n', '').split(',')
73+
# Cleans up empty bits as a result of parsing difficulties.
74+
new = [x.strip() for x in new if x.strip()]
75+
old = [x.strip() for x in old if x.strip()]
76+
all_eclasses = set()
77+
old_vars = {}
78+
new_vars = {}
79+
current_eclass = ''
80+
for item in old:
81+
if '--' in item:
82+
# It's an eclass comment line.
83+
current_eclass = item[3:]
84+
all_eclasses.add(current_eclass)
85+
continue
86+
else:
87+
# It's a var, so add it to the dict of the current eclass.
88+
old_vars.setdefault(current_eclass, []).append(item)
89+
for item in new:
90+
if '--' in item:
91+
# It's an eclass comment line.
92+
current_eclass = item[3:]
93+
all_eclasses.add(current_eclass)
94+
continue
95+
else:
96+
# It's a var, so add it to the dict of the current eclass.
97+
new_vars.setdefault(current_eclass, []).append(item)
98+
for eclass in sorted(all_eclasses):
99+
if eclass in old_vars:
100+
if eclass not in new_vars:
101+
# Checks if the entire eclass is removed.
102+
print(f'{eclass} not present in new variables.')
103+
for var in old_vars[eclass]:
104+
print(f'\t-{var}')
105+
print()
106+
else:
107+
# Eclass isn't removed, so check for added or removed vars.
108+
toprint = '\n'.join(
109+
[f'\t-{x}' for x in old_vars[eclass] if x not in new_vars[eclass]] +
110+
[f'\t+{x}' for x in new_vars[eclass] if x not in old_vars[eclass]])
111+
if toprint:
112+
print(eclass)
113+
print(toprint)
114+
if eclass in new_vars:
115+
if eclass not in old_vars:
116+
# Checks if entire eclass is new.
117+
print(f'{eclass} added in new variables.')
118+
for var in new_vars[eclass]:
119+
print(f'\t+{var}')
120+
print()

shellcheck.1.md

+3-2
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,9 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts.
8787

8888
: Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash* and *ksh*.
8989
The default is to deduce the shell from the file's `shell` directive,
90-
shebang, or `.bash/.bats/.dash/.ksh` extension, in that order. *sh* refers to
91-
POSIX `sh` (not the system's), and will warn of portability issues.
90+
shebang, or `.bash/.bats/.dash/.ksh/.ebuild/.eclass` extension, in that
91+
order. *sh* refers to POSIX `sh` (not the system's), and will warn of
92+
portability issues.
9293

9394
**-S**\ *SEVERITY*,\ **--severity=***severity*
9495

src/ShellCheck/ASTLib.hs

+18-2
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,6 @@ import Numeric (showHex)
3636

3737
import Test.QuickCheck
3838

39-
arguments (T_SimpleCommand _ _ (cmd:args)) = args
40-
4139
-- Is this a type of loop?
4240
isLoop t = case t of
4341
T_WhileExpression {} -> True
@@ -559,11 +557,29 @@ getCommandNameFromExpansion t =
559557
extract (T_Pipeline _ _ [cmd]) = getCommandName cmd
560558
extract _ = Nothing
561559

560+
-- If a command substitution is a single command, get its argument Tokens.
561+
-- Return an empty list if there are no arguments or the token is not a command substitution.
562+
-- $(date +%s) = ["+%s"]
563+
getArgumentsFromExpansion :: Token -> [Token]
564+
getArgumentsFromExpansion t =
565+
case t of
566+
T_DollarExpansion _ [c] -> extract c
567+
T_Backticked _ [c] -> extract c
568+
T_DollarBraceCommandExpansion _ [c] -> extract c
569+
_ -> []
570+
where
571+
extract (T_Pipeline _ _ [cmd]) = arguments cmd
572+
extract _ = []
573+
562574
-- Get the basename of a token representing a command
563575
getCommandBasename = fmap basename . getCommandName
564576

565577
basename = reverse . takeWhile (/= '/') . reverse
566578

579+
-- Get the arguments to a command
580+
arguments (T_SimpleCommand _ _ (cmd:args)) = args
581+
arguments t = maybe [] arguments (getCommand t)
582+
567583
isAssignment t =
568584
case t of
569585
T_Redirecting _ _ w -> isAssignment w

0 commit comments

Comments
 (0)