"""Parsing of C and C++ commands and extraction of search paths."""
__author__ = "opensource@google.com (Craig Silverstein, Nils Klarlund)"
import re
import os
import sys
import basics
import cache_basics
Debug = basics.Debug
DEBUG_TRACE = basics.DEBUG_TRACE
NotCoveredError = basics.NotCoveredError
class ParseState:
"""Everything we figure out during parsing. This is accessed a lot and
needs to be fast, so you should access and set the data members directly.
Mutator functions are provided for the non-list elements, but solely
because this way you can set these elements from within a lambda.
"""
def __init__(self):
self.nostdinc = False
self.file_names = []
self.quote_dirs = []
self.include_files = []
self.include_dirs = []
self.before_system_dirs = []
self.after_system_dirs = []
self.language = 'none' self.isysroot = None
self.sysroot = None
self.output_file = None
self.iprefix = ""
self.Dopts = []
def set_nostdinc(self): self.nostdinc = True
def set_language(self, x): self.language = x
def set_isysroot(self, x): self.isysroot = x
def set_sysroot(self, x):
if x[0] == '=':
self.sysroot = x[1:]
else:
self.sysroot = x
def set_outputfile(self, x): self.output_file = x
def set_iprefix(self, x): self.iprefix = x
def SysRootInfo(self):
if self.isysroot:
return ( "-isysroot " + self.isysroot, [ "-isysroot", self.isysroot ] )
if self.sysroot:
return ( "--sysroot=" + self.sysroot, [ "--sysroot=" + self.sysroot ] )
return ( "", [] )
def _SplitMacroArg(arg):
"""Split an arg as found in -Darg
Argument:
arg: argument
Returns: [arg] if there is no '=' in arg, otherwise [symb, val], where symb is
what is to the left of '=' and val is what is to the right.
"""
pos = arg.find("=")
if pos > 0:
return [arg[:pos], arg[pos + 1:]]
else:
return [arg]
def _RaiseNotImplemented(name, comment=''):
raise NotCoveredError('%s is not implemented. %s' % (name, comment))
CPP_OPTIONS_MAYBE_TWO_WORDS = {
'-MF': lambda ps, arg: None,
'-MT': lambda ps, arg: None,
'-MQ': lambda ps, arg: None,
'-arch': lambda ps, arg: None,
'-iframework': lambda ps, arg: ps.include_dirs.append((arg,
basics.INCLUDE_DIR_FRAMEWORKS)),
'-include': lambda ps, arg: ps.include_files.append(arg),
'-imacros': lambda ps, arg: ps.include_files.append(arg),
'-idirafter': lambda ps, arg: ps.after_system_dirs.append(arg),
'-iprefix': lambda ps, arg: ps.set_iprefix(arg),
'-iwithprefix': lambda ps, arg: ps.after_system_dirs.append(
os.path.join(ps.iprefix, arg)),
'-iwithprefixbefore': lambda ps, arg: ps.include_dirs.append(
(os.path.join(ps.iprefix, arg),
basics.INCLUDE_DIR_NORMAL)),
'-isysroot': lambda ps, arg: ps.set_isysroot(arg),
'-imultilib': lambda ps, arg: _RaiseNotImplemented('-imultilib'),
'-isystem': lambda ps, arg: ps.before_system_dirs.append(arg),
'-iquote': lambda ps, arg: ps.quote_dirs.append(arg),
'--sysroot': lambda ps, arg: ps.set_sysroot(arg),
}
CPP_OPTIONS_MAYBE_TWO_WORDS_FIRST_LETTERS = ('M', 'a', 'i', '-')
for key in CPP_OPTIONS_MAYBE_TWO_WORDS.keys():
assert key[1] in CPP_OPTIONS_MAYBE_TWO_WORDS_FIRST_LETTERS
CPP_OPTIONS_ALWAYS_TWO_WORDS = {
'-Xpreprocessor': lambda ps, arg: _RaiseNotImplemented('-Xpreprocessor'),
'-aux-info': lambda ps, arg: None,
'--param': lambda ps, arg: None,
'-Xassembler': lambda ps, arg: None,
'-Xlinker': lambda ps, arg: None,
}
CPP_OPTIONS_TWO_WORDS = {}
CPP_OPTIONS_TWO_WORDS.update(CPP_OPTIONS_MAYBE_TWO_WORDS)
CPP_OPTIONS_TWO_WORDS.update(CPP_OPTIONS_ALWAYS_TWO_WORDS)
CPP_OPTIONS_ONE_WORD = {
'-undef': lambda ps, arg: None,
'-nostdinc': lambda ps: ps.set_nostdinc(),
}
CPP_OPTIONS_ONE_LETTER = {
'D': lambda ps, arg: ps.Dopts.append(arg.split('=')),
'F': lambda ps, arg: ps.include_dirs.append((arg,
basics.INCLUDE_DIR_FRAMEWORKS)),
'I': lambda ps, arg: ps.include_dirs.append((arg,
basics.INCLUDE_DIR_NORMAL)),
'U': lambda ps, arg: None,
'o': lambda ps, arg: ps.set_outputfile(arg),
'x': lambda ps, arg: ps.set_language(arg),
'A': lambda ps, arg: None,
'l': lambda ps, arg: None,
'u': lambda ps, arg: None,
'L': lambda ps, arg: None,
'B': lambda ps, arg: None,
'V': lambda ps, arg: None,
'b': lambda ps, arg: None,
}
NONSPACE_RE = re.compile(r'\S') SPACE_RE = re.compile(r'\s')
NONESC_QUOTE_RE = re.compile(r'[^\\]"|^"') QUOTE_RE = re.compile(r'(?<!\\)"') ESC_QUOTE_RE = re.compile(r'\\"')
def ParseCommandLineSlowly(line):
"""Parse line as if it were issued in a shell.
Split the line into a list of string arguments indicated by spaces,
except that doubly quoted substrings are treated atomically. Also,
do allow backslash escaped quotes; they are turned into regular
quotes. This function is written for efficiency; only very simple
regular expressions are used in main loop.
The parser is not needed when the include server is driven by
distcc, because the distcc client passes the argv vector. It is used
as part of a faster parser.
"""
if "'" in line:
raise NotCoveredError("Single-quotes not accepted in command line.")
args = []
m_unesc_q = NONESC_QUOTE_RE.search(line, 0)
if m_unesc_q:
unesc_q = m_unesc_q.end() - 1
else:
unesc_q = sys.maxint
m_nonspc = NONSPACE_RE.search(line, 0)
if not m_nonspc:
return args
start = m_nonspc.start()
end = start + 1
while True:
assert start <= unesc_q
assert start < end <= len(line), (start, end, len(line))
assert not SPACE_RE.match(line, start)
assert unesc_q == sys.maxint or line[unesc_q] == '"'
try:
end = SPACE_RE.search(line, end).start()
except AttributeError:
end = len(line)
if end < unesc_q:
args.append(ESC_QUOTE_RE.sub(
'"',
QUOTE_RE.sub(
'',
line[start:end])))
try:
start = NONSPACE_RE.search(line, end).start()
except AttributeError:
return args
end = start + 1
continue
assert start <= unesc_q < end
if unesc_q == len(line) - 1:
raise NotCoveredError("""Unexpected '"' at end of line.""")
m_unesc_q = NONESC_QUOTE_RE.search(line, unesc_q + 1)
if not m_unesc_q:
raise NotCoveredError("""Missing '"', could not parse command line.""")
assert m_unesc_q.end() - 1 > unesc_q
end = m_unesc_q.end()
if end == len(line):
args.append(ESC_QUOTE_RE.sub(
'"',
QUOTE_RE.sub(
'',
line[start:end])))
return args
m_unesc_q = NONESC_QUOTE_RE.search(line, end)
if m_unesc_q:
unesc_q = m_unesc_q.end() - 1
else:
unesc_q = sys.maxint
def ParseCommandLine(line):
"""Parse line as it were issued in a shell (optimized).
"""
quote_pos = line.rfind('"')
if quote_pos == -1:
return line.split()
else:
good_pos = line.find(' ', quote_pos)
if good_pos != -1:
return (ParseCommandLineSlowly(line[0:good_pos])
+ line[good_pos:].split())
else: return ParseCommandLineSlowly(line)
TRANSLATION_UNIT_FILEPATH_RE = (
re.compile(r".*[.](?P<suffix>%s)$" %
'|'.join([re.escape(ext)
for ext in basics.TRANSLATION_UNIT_MAP.keys()])))
def ParseCommandArgs(args, current_dir, includepath_map, dir_map,
compiler_defaults, timer=None):
"""Parse arguments like -I to make include directory lists.
Arguments:
args: list of arguments (strings)
current_dir: string
includepath_map: a MapToIndex object
dir_map: a DirectoryMapToIndex object
compiler_defaults: a CompilerDefaults object
timer: a basics.IncludeAnalyzerTimer object
Returns:
(quote_dirs, angle_dirs, files, source_file, source_file_prefix, dopts)
where:
quote_dirs: a list of dir_map-indexed directories
angle_dirs: a list of dir_map-indexed directories
files: a list of includepath_map-indexed files
source_file_prefix: the source file name with extension stripped
dopts: a list of items as returned by _SplitMacroArg
Modifies:
compiler_defaults
"""
if __debug__: Debug(DEBUG_TRACE, "ParseCommand %s" % args)
assert isinstance(dir_map, cache_basics.DirectoryMapToIndex)
assert isinstance(includepath_map, cache_basics.MapToIndex)
parse_state = ParseState()
if len(args) < 2:
raise NotCoveredError("Command line: too few arguments.")
compiler = args[0]
i = 1
while i < len(args):
if args[i][0] != '-' or args[i] == '-': if args[i].startswith('"-'):
pass else:
parse_state.file_names.append(args[i]) i += 1
continue
action = CPP_OPTIONS_ONE_LETTER.get(args[i][1]) if action:
arg = args[i][2:]
if arg: action(parse_state, arg)
i += 1
else: try:
action(parse_state, args[i+1])
i += 2
except IndexError:
raise NotCoveredError("No argument found for option '%s'" % args[i])
continue
action = CPP_OPTIONS_TWO_WORDS.get(args[i])
if action:
try:
action(parse_state, args[i+1])
i += 2
except IndexError:
raise NotCoveredError("No argument found for option '%s'" % args[i])
continue
action = CPP_OPTIONS_ONE_WORD.get(args[i])
if action:
action(parse_state)
i += 1
continue
if args[i][1] in CPP_OPTIONS_MAYBE_TWO_WORDS_FIRST_LETTERS: found_action = False
for (option, action) in CPP_OPTIONS_MAYBE_TWO_WORDS.items():
if action and args[i].startswith(option):
action(parse_state, args[i][len(option):])
i += 1
found_action = True
break
if found_action: continue
i += 1
continue
for (d, t) in parse_state.include_dirs:
if d == "-":
_RaiseNotImplemented('-I-', '(Use -iquote instead.)')
if len(parse_state.file_names) != 1:
raise NotCoveredError(
"Could not locate name of translation unit: %s." % parse_state.file_names,
send_email=False)
source_file = parse_state.file_names[0]
if parse_state.output_file:
source_file_prefix = re.sub("[.]o$", "", parse_state.output_file)
else:
source_file_prefix = re.sub("[.](%s)$" %
"|".join(basics.TRANSLATION_UNIT_MAP.keys()),
"",
source_file)
source_file_prefix = os.path.join(current_dir, source_file_prefix)
if parse_state.language == 'none': language_match = TRANSLATION_UNIT_FILEPATH_RE.match(source_file)
if not language_match:
raise NotCoveredError(
"For source file '%s': unrecognized filename extension" % source_file)
suffix = language_match.group('suffix')
parse_state.language = basics.TRANSLATION_UNIT_MAP[suffix]
assert parse_state.language in basics.LANGUAGES
compiler_defaults.SetSystemDirsDefaults(compiler, parse_state.language,
parse_state.SysRootInfo(), timer)
def IndexDirs(dir_list):
"""Normalize directory names and index, but the list of names is actually
pairs of the name and type of directory (normal vs. framework).
Remove leading "./" and trailing "/"'s from directory paths in
dir_list before indexing them according to dir_map.
"""
S = basics.SafeNormPath
I = dir_map.Index
idx_list = []
for (d, t) in dir_list:
d = S(d)
if t == basics.INCLUDE_DIR_NORMAL:
idx_list.append(I(d))
else:
assert t == basics.INCLUDE_DIR_FRAMEWORKS
idx_list.append(I('*H' + d))
idx_list.append(I('*P' + d))
return idx_list
angle_dirs = IndexDirs(parse_state.include_dirs)
angle_dirs.extend(IndexDirs([(d, basics.INCLUDE_DIR_NORMAL) for d in
parse_state.before_system_dirs]))
if not parse_state.nostdinc:
angle_dirs.extend(
IndexDirs(compiler_defaults.system_dirs_default
[compiler][parse_state.language][parse_state.SysRootInfo()[0]]))
angle_dirs.extend(IndexDirs([(d, basics.INCLUDE_DIR_NORMAL) for d in
parse_state.after_system_dirs]))
quote_dirs = IndexDirs([(d, basics.INCLUDE_DIR_NORMAL) for d in
parse_state.quote_dirs])
quote_dirs.extend(angle_dirs)
angle_dirs = tuple(angle_dirs)
quote_dirs = tuple(quote_dirs)
include_files = tuple(
[includepath_map.Index(basics.SafeNormPath(f),
ignore_absolute_path_warning=True)
for f in parse_state.include_files])
if __debug__: Debug(DEBUG_TRACE, ("ParseCommand result: %s %s %s %s %s %s" %
(quote_dirs, angle_dirs, include_files,
source_file, source_file_prefix,
parse_state.Dopts)))
return (quote_dirs, angle_dirs, include_files, source_file, source_file_prefix,
parse_state.Dopts)