"""Common and low-level stuff for include server."""
__author__ = 'Nils Klarlund'
import glob
import os.path
import resource
import signal
import shutil
import sys
import tempfile
class ClientRootKeeper(object):
"""Determine the tmp directory to use for compressed files.
Use the RAM disk-like /dev/shm as default place to store compressed files if
available. The protocol between the include server and distcc client
stipulates that the top three directories constitute the prefix prepended to
absolute file paths.
Instance vars:
client_tmp: a path, the place for creation of temporary directories.
client_root: a path, the current such temporary directory
_client_root_before_padding: a path kept for testing purposes
A typical client root looks like:
- /tmp/tmpBDoZQV.include_server-6642-13/padding, or
- /dev/shm/tmpBDoZQV.include_server-6642-19
Note that each path has exactly three directory components to it. This is an
invariant. Some client roots are padded with '/padding' to satisfy the
invariant.
"""
INCLUDE_SERVER_NAME = 'include_server'
def __init__(self):
"""Constructor."""
if 'DISTCC_CLIENT_TMP' in os.environ:
self.client_tmp = os.environ['DISTCC_CLIENT_TMP']
elif os.path.isdir('/dev/shm') and os.access('/dev/shm',
os.X_OK + os.W_OK + os.R_OK):
self.client_tmp = '/dev/shm'
else:
self.client_tmp = '/tmp'
if not self.client_tmp or self.client_tmp[0] != '/':
sys.exit("""DISTCC_CLIENT_TMP must start with '/'.""")
self.client_tmp = self.client_tmp.rstrip('/')
if len(self.client_tmp.split('/')) > 3:
sys.exit('DISTCC_CLIENT_TMP must have at most two directory levels.')
self.number_missing_levels = 3 - len(self.client_tmp.split('/'))
self.client_root = None
def Glob(self, pid_expr):
"""Glob unpadded client roots whose pid is matched by pid expression."""
return glob.glob('%s/*.%s-%s-*'
% (self.client_tmp, self.INCLUDE_SERVER_NAME,
pid_expr))
def ClientRootMakedir(self, generation):
"""Make a new client directory for a generation of compressed files.
Arguments:
generation: a natural number, usually 1 or slightly bigger; this number,
minus 1, indicates how many times a reset of the caches has taken place.
"""
try:
self._client_root_before_padding = tempfile.mkdtemp(
'.%s-%s-%d' %
(self.INCLUDE_SERVER_NAME,
os.getpid(), generation),
dir=self.client_tmp)
self.client_root = (self._client_root_before_padding
+ '/padding' * self.number_missing_levels)
if not os.path.isdir(self.client_root):
os.makedirs(self.client_root)
except (IOError, OSError), why:
sys.exit('Could not create client root directory %s: %s' %
(self.client_root, why))
def CleanOutClientRoots(self, pid=None):
"""Delete client root directories pertaining to this process.
Args:
pid: None (which means 'pid of current process') or an integer
"""
if not pid:
pid = os.getpid()
for client_root in self.Glob(str(pid)):
shutil.rmtree(client_root, ignore_errors=True)
def CleanOutOthers(self):
"""Search for left-overs from include servers that have passed away."""
distcc_directories = self.Glob('*')
for directory in distcc_directories:
hyphen_ultimate_position = directory.rfind('-')
assert hyphen_ultimate_position != -1
hyphen_penultimate_position = directory.rfind('-', 0,
hyphen_ultimate_position)
assert hyphen_penultimate_position != -1
pid_str = directory[hyphen_penultimate_position + 1:
hyphen_ultimate_position]
try:
pid = int(pid_str)
except ValueError:
continue try:
os.getpgid(pid)
continue
except OSError:
if not os.access(directory, os.W_OK):
continue Debug(DEBUG_TRACE,
"Cleaning out '%s' after defunct include server." % directory)
self.CleanOutClientRoots(pid)
MAX_COMPONENTS_IN_SERVER_ROOT = 20
DCC_EMAILLOG_WHOM_TO_BLAME = os.getenv('DISTCC_EMAILLOG_WHOM_TO_BLAME',
'distcc-pump-errors')
EMAIL_SUBJECT = 'distcc-pump include server email'
CANT_SEND_MESSAGE = """Please notify %s that the distcc-pump include server
tried to send them email but failed.""" % DCC_EMAILLOG_WHOM_TO_BLAME
MAX_EMAILS_TO_SEND = 3
USER_TIME_QUOTA = 6.0
USER_TIME_QUOTA_CHECK_INTERVAL_TIME = 4
SIMPLE = 0 MEMOIZING = 1 ALGORITHMS = [SIMPLE, MEMOIZING]
GC_THRESHOLD = 10000
INCLUDE_DIR_NORMAL = 1 INCLUDE_DIR_FRAMEWORKS = 2
opt_algorithm = MEMOIZING opt_debug_pattern = 1 opt_email_bound = MAX_EMAILS_TO_SEND
opt_exact_analysis = False opt_print_times = False
opt_path_observation_re = None
opt_send_email = False
opt_simple_algorithm = False
opt_stat_reset_triggers = {}
opt_statistics = False
opt_unsafe_absolute_includes = False
opt_verify = False opt_write_include_closure = False
def Stamp(path):
"""Return a stamp characterizing a file and its modification time."""
try:
st_inf = os.stat(path)
return (st_inf.st_mtime, st_inf.st_ino, st_inf.st_dev)
except OSError:
return None
LANGUAGES = set(['c', 'c++', 'objective-c', 'objective-c++'])
TRANSLATION_UNIT_MAP = {
'c': 'c', 'i': 'c',
'cc': 'c++', 'cpp': 'c++', 'cxx': 'c++', 'C': 'c++', 'CXX': 'c++',
'ii': 'c++',
'm': 'objective-c', 'mi': 'objective-c',
'mm': 'objective-c++', 'M': 'objective-c++', 'mii': 'objective-c++',
}
assert set(TRANSLATION_UNIT_MAP.values()) == LANGUAGES
DEBUG_WARNING = 1 DEBUG_TRACE = 2 DEBUG_TRACE1 = 4 DEBUG_TRACE2 = 8 DEBUG_DATA = 16 DEBUG_NUM_BITS = 5
def Debug(trigger_pattern, message, *params):
"""Print message to stderr depending on trigger pattern.
Args:
trigger_pattern: a bit vector (as an integer)
message: a format string
params: arguments to message
"""
triggered = opt_debug_pattern & trigger_pattern
if triggered:
i = 1
for unused_j in range(DEBUG_NUM_BITS):
if i & DEBUG_WARNING & triggered:
print >> sys.stderr, 'WARNING include server:', message % params
if i & DEBUG_TRACE & triggered:
print >> sys.stderr, 'TRACE:', message % params
elif i & DEBUG_TRACE1 & triggered:
print >> sys.stderr, 'TRACE1:', message % params
elif i & DEBUG_TRACE2 & triggered:
print >> sys.stderr, 'TRACE2:', message % params
elif i & DEBUG_DATA & triggered:
print >> sys.stderr, 'DATA:', message % params
i *= 2
sys.stderr.flush()
class Error(Exception):
"""For include server errors."""
pass
class NotCoveredError(Error):
"""Exception for included file not covered by include processing."""
def __init__(self, message,
source_file=None,
line_number=None,
send_email=True):
"""Constructor.
Arguments:
message: text of error message
source_file: name of source_file if known
line_number: an integer, if known
send_email: a Boolean, if False then never send email
These arguments are all stored in the exception. However, the source_file
and line_number are appended, in a syntax defined here, to the message
before it is stored as self.args[0] through invocation of the Error
constructor.
"""
assert not line_number or source_file
self.source_file = None
self.line_number = None
self.send_email = send_email
if source_file:
self.source_file = source_file
if line_number:
self.line_number = line_number
message = ("""File: '%s', line: %s: %s"""
% (source_file, line_number, message))
else:
message = """File: '%s': %s""" % (source_file, message)
Error.__init__(self, message)
class NotCoveredTimeOutError(NotCoveredError):
"""Raised when spending too much time analyzing dependencies."""
pass
class IncludeAnalyzerTimer(object):
"""Start a timer limiting CPU time for servicing a single request.
We use user time so that a network hiccup will not entail a cache reset if,
say, we are using NFS.
An object of this class must be instantiated so that, no matter what, the
Cancel method is eventually called. This reinstates the original timer (if
present).
"""
def __init__(self):
self.start_utime = resource.getrusage(resource.RUSAGE_SELF).ru_utime
self.old = signal.signal(signal.SIGALRM, self._TimeIsUp)
signal.alarm(USER_TIME_QUOTA_CHECK_INTERVAL_TIME)
def _TimeIsUp(self, unused_sig_number, unused_frame):
"""Check CPU time spent and raise exception or reschedule."""
if (resource.getrusage(resource.RUSAGE_SELF).ru_utime
> self.start_utime + USER_TIME_QUOTA):
raise NotCoveredTimeOutError(('Bailing out because include server '
+ 'spent more than %3.1fs user time '
+ 'handling request') %
USER_TIME_QUOTA)
else:
signal.alarm(USER_TIME_QUOTA_CHECK_INTERVAL_TIME)
def Stop(self):
signal.alarm(0)
def Start(self):
signal.alarm(USER_TIME_QUOTA_CHECK_INTERVAL_TIME)
def Cancel(self):
"""Must be called eventually. See class documentation."""
sys.stdout.flush()
signal.alarm(0)
signal.signal(signal.SIGALRM, self.old)
class SignalSIGTERM(Error):
pass
def RaiseSignalSIGTERM(*unused_args):
"""Raise SignalSIGTERM.
Use signal.signal for binding this function to SIGTERM.
"""
raise SignalSIGTERM
def SafeNormPath(path):
"""Safe, but limited, version of os.path.normpath.
Args:
path: a string
Returns:
a string
Python's os.path.normpath is an unsafe operation; the result may not point to
the same file as the argument. Instead, this function just removes
initial './'s and a final '/'s if present.
"""
if path == '.':
return ''
else:
while path.startswith('./'):
path = path[2:]
return path.rstrip('/')
def PathFromDirMapEntryAndInclude(dirmap_entry, leaf):
"""Takes a DirectoryMap entry and leaf path and builds a full path for it.
DirectoryMap entries already have a trailing slash, so the strings
can just be appended together. *BUT* when we put framework search paths
into the DirectoryMap, we list them twice prefixed with "*H" and "*P" to
tell the rest of the code that the leaf must be processed according to
framework rules to build the path.
If the leaf is invalid for a framework search, and dirmap_entry is
a framework search directory, None is returned.
"""
if not dirmap_entry or dirmap_entry[0] != '*':
return dirmap_entry + leaf
if not '/' in leaf:
return None
(i_fwk, i_hdr) = leaf.split('/', 1)
i_fwk = i_fwk + '.framework/'
if dirmap_entry[1] == 'H':
return dirmap_entry[2:] + i_fwk + 'Headers/' + i_hdr
assert dirmap_entry[1] == 'P'
return dirmap_entry[2:] + i_fwk + 'PrivateHeaders/' + i_hdr