import os
import re
import sys
import glob
import fnmatch
import shutil
import urllib2
import hashlib
import tarfile
import logging
import datetime
import tempfile
import operator
import itertools
import subprocess
import argparse
try:
import ezt
except ImportError:
ezt_path = os.path.dirname(os.path.dirname(os.path.abspath(sys.path[0])))
ezt_path = os.path.join(ezt_path, 'build', 'generator')
sys.path.append(ezt_path)
import ezt
sys.path.remove(ezt_path)
tool_versions = {
'trunk' : {
'autoconf' : ['2.69',
'954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'],
'libtool' : ['2.4.6',
'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'],
'swig' : ['3.0.10',
'2939aae39dec06095462f1b95ce1c958ac80d07b926e48871046d17c0094f44c'],
},
'1.10' : {
'autoconf' : ['2.69',
'954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'],
'libtool' : ['2.4.6',
'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'],
'swig' : ['3.0.10',
'2939aae39dec06095462f1b95ce1c958ac80d07b926e48871046d17c0094f44c'],
},
'1.9' : {
'autoconf' : ['2.69',
'954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'],
'libtool' : ['2.4.6',
'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'],
'swig' : ['2.0.12',
'65e13f22a60cecd7279c59882ff8ebe1ffe34078e85c602821a541817a4317f7'],
},
'1.8' : {
'autoconf' : ['2.69',
'954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'],
'libtool' : ['2.4.3',
'36b4881c1843d7585de9c66c4c3d9a067ed3a3f792bc670beba21f5a4960acdf'],
'swig' : ['2.0.9',
'586954000d297fafd7e91d1ad31089cc7e249f658889d11a44605d3662569539'],
},
}
recommended_release = '1.9'
repos = 'https://svn.apache.org/repos/asf/subversion'
secure_repos = 'https://svn.apache.org/repos/asf/subversion'
dist_repos = 'https://dist.apache.org/repos/dist'
dist_dev_url = dist_repos + '/dev/subversion'
dist_release_url = dist_repos + '/release/subversion'
KEYS = 'https://people.apache.org/keys/group/subversion.asc'
extns = ['zip', 'tar.gz', 'tar.bz2']
class Version(object):
regex = re.compile(r'(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?')
def __init__(self, ver_str):
if ver_str == 'trunk-nightly':
self.major = None
self.minor = None
self.patch = None
self.pre = 'nightly'
self.pre_num = None
self.base = 'nightly'
self.branch = 'trunk'
return
match = self.regex.search(ver_str)
if not match:
raise RuntimeError("Bad version string '%s'" % ver_str)
self.major = int(match.group(1))
self.minor = int(match.group(2))
self.patch = int(match.group(3))
if match.group(4):
self.pre = match.group(4)
self.pre_num = int(match.group(5))
else:
self.pre = None
self.pre_num = None
self.base = '%d.%d.%d' % (self.major, self.minor, self.patch)
self.branch = '%d.%d' % (self.major, self.minor)
def is_prerelease(self):
return self.pre != None
def is_recommended(self):
return self.branch == recommended_release
def get_download_anchor(self):
if self.is_prerelease():
return 'pre-releases'
else:
if self.is_recommended():
return 'recommended-release'
else:
return 'supported-releases'
def get_ver_tags(self, revnum):
ver_tag = ''
ver_numtag = ''
if self.pre == 'alpha':
ver_tag = '" (Alpha %d)"' % self.pre_num
ver_numtag = '"-alpha%d"' % self.pre_num
elif self.pre == 'beta':
ver_tag = '" (Beta %d)"' % args.version.pre_num
ver_numtag = '"-beta%d"' % self.pre_num
elif self.pre == 'rc':
ver_tag = '" (Release Candidate %d)"' % self.pre_num
ver_numtag = '"-rc%d"' % self.pre_num
elif self.pre == 'nightly':
ver_tag = '" (Nightly Build r%d)"' % revnum
ver_numtag = '"-nightly-r%d"' % revnum
else:
ver_tag = '" (r%d)"' % revnum
ver_numtag = '""'
return (ver_tag, ver_numtag)
def __serialize(self):
return (self.major, self.minor, self.patch, self.pre, self.pre_num)
def __eq__(self, that):
return self.__serialize() == that.__serialize()
def __ne__(self, that):
return self.__serialize() != that.__serialize()
def __hash__(self):
return hash(self.__serialize())
def __lt__(self, that):
if self.major < that.major: return True
if self.major > that.major: return False
if self.minor < that.minor: return True
if self.minor > that.minor: return False
if self.patch < that.patch: return True
if self.patch > that.patch: return False
if not self.pre and not that.pre: return False
if not self.pre and that.pre: return False
if self.pre and not that.pre: return True
if self.pre != that.pre:
return self.pre < that.pre
else:
return self.pre_num < that.pre_num
def __str__(self):
"Return an SVN_VER_NUMBER-formatted string, or 'nightly'."
if self.pre:
if self.pre == 'nightly':
return 'nightly'
else:
extra = '-%s%d' % (self.pre, self.pre_num)
else:
extra = ''
return self.base + extra
def __repr__(self):
return "Version(%s)" % repr(str(self))
def get_prefix(base_dir):
return os.path.join(base_dir, 'prefix')
def get_tempdir(base_dir):
return os.path.join(base_dir, 'tempdir')
def get_workdir(base_dir):
return os.path.join(get_tempdir(base_dir), 'working')
def get_exportdir(base_dir, version, revnum):
if version.pre != 'nightly':
return os.path.join(get_tempdir(base_dir), 'subversion-'+str(version))
return os.path.join(get_tempdir(base_dir),
'subversion-%s-r%d' % (version, revnum))
def get_deploydir(base_dir):
return os.path.join(base_dir, 'deploy')
def get_target(args):
"Return the location of the artifacts"
if args.target:
return args.target
else:
return get_deploydir(args.base_dir)
def get_tmpldir():
return os.path.join(os.path.abspath(sys.path[0]), 'templates')
def get_tmplfile(filename):
try:
return open(os.path.join(get_tmpldir(), filename))
except IOError:
return urllib2.urlopen(repos + '/trunk/tools/dist/templates/' + filename)
def get_nullfile():
return open(os.path.devnull, 'w')
def run_script(verbose, script, hide_stderr=False):
stderr = None
if verbose:
stdout = None
else:
stdout = get_nullfile()
if hide_stderr:
stderr = get_nullfile()
for l in script.split('\n'):
subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)
def download_file(url, target, checksum):
response = urllib2.urlopen(url)
target_file = open(target, 'w+')
target_file.write(response.read())
target_file.seek(0)
m = hashlib.sha256()
m.update(target_file.read())
target_file.close()
checksum2 = m.hexdigest()
if checksum != checksum2:
raise RuntimeError("Checksum mismatch for '%s': "\
"downloaded: '%s'; expected: '%s'" % \
(target, checksum, checksum2))
ezt_False = ""
ezt_True = True
def ezt_bool(boolean_value):
return ezt_True if boolean_value else ezt_False
def cleanup(args):
'Remove generated files and folders.'
logging.info('Cleaning')
shutil.rmtree(get_prefix(args.base_dir), True)
shutil.rmtree(get_tempdir(args.base_dir), True)
shutil.rmtree(get_deploydir(args.base_dir), True)
class RollDep(object):
'The super class for each of the build dependencies.'
def __init__(self, base_dir, use_existing, verbose):
self._base_dir = base_dir
self._use_existing = use_existing
self._verbose = verbose
def _test_version(self, cmd):
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
(stdout, stderr) = proc.communicate()
rc = proc.wait()
if rc: return ''
return stdout.split('\n')
def build(self):
if not hasattr(self, '_extra_configure_flags'):
self._extra_configure_flags = ''
cwd = os.getcwd()
tempdir = get_tempdir(self._base_dir)
tarball = os.path.join(tempdir, self._filebase + '.tar.gz')
if os.path.exists(tarball):
if not self._use_existing:
raise RuntimeError('autoconf tarball "%s" already exists'
% tarball)
logging.info('Using existing %s.tar.gz' % self._filebase)
else:
logging.info('Fetching %s' % self._filebase)
download_file(self._url, tarball, self._checksum)
tarfile.open(tarball).extractall(tempdir)
logging.info('Building ' + self.label)
os.chdir(os.path.join(tempdir, self._filebase))
run_script(self._verbose,
'''./configure --prefix=%s %s
make
make install''' % (get_prefix(self._base_dir),
self._extra_configure_flags))
os.chdir(cwd)
class AutoconfDep(RollDep):
def __init__(self, base_dir, use_existing, verbose, autoconf_ver, checksum):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'autoconf'
self._filebase = 'autoconf-' + autoconf_ver
self._autoconf_ver = autoconf_ver
self._url = 'https://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase
self._checksum = checksum
def have_usable(self):
output = self._test_version(['autoconf', '-V'])
if not output: return False
version = output[0].split()[-1:][0]
return version == self._autoconf_ver
def use_system(self):
if not self._use_existing: return False
return self.have_usable()
class LibtoolDep(RollDep):
def __init__(self, base_dir, use_existing, verbose, libtool_ver, checksum):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'libtool'
self._filebase = 'libtool-' + libtool_ver
self._libtool_ver = libtool_ver
self._url = 'https://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase
self._checksum = checksum
def have_usable(self):
output = self._test_version(['libtool', '--version'])
if not output: return False
return self._libtool_ver in output[0]
def use_system(self):
return False
def build(self):
RollDep.build(self)
bin_dir = os.path.join(get_prefix(self._base_dir), "bin")
os.symlink("libtoolize", os.path.join(bin_dir, "glibtoolize"))
os.symlink("libtool", os.path.join(bin_dir, "glibtool"))
class SwigDep(RollDep):
def __init__(self, base_dir, use_existing, verbose, swig_ver, checksum,
sf_mirror):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'swig'
self._filebase = 'swig-' + swig_ver
self._swig_ver = swig_ver
self._url = 'https://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
{ 'swig' : self._filebase,
'sf_mirror' : sf_mirror }
self._checksum = checksum
self._extra_configure_flags = '--without-pcre'
def have_usable(self):
output = self._test_version(['swig', '-version'])
if not output: return False
version = output[1].split()[-1:][0]
return version == self._swig_ver
def use_system(self):
if not self._use_existing: return False
return self.have_usable()
def build_env(args):
'Download prerequisites for a release and prepare the environment.'
logging.info('Creating release environment')
try:
os.mkdir(get_prefix(args.base_dir))
os.mkdir(get_tempdir(args.base_dir))
except OSError:
if not args.use_existing:
raise
autoconf = AutoconfDep(args.base_dir, args.use_existing, args.verbose,
tool_versions[args.version.branch]['autoconf'][0],
tool_versions[args.version.branch]['autoconf'][1])
libtool = LibtoolDep(args.base_dir, args.use_existing, args.verbose,
tool_versions[args.version.branch]['libtool'][0],
tool_versions[args.version.branch]['libtool'][1])
swig = SwigDep(args.base_dir, args.use_existing, args.verbose,
tool_versions[args.version.branch]['swig'][0],
tool_versions[args.version.branch]['swig'][1],
args.sf_mirror)
for dep in [autoconf, libtool, swig]:
if dep.use_system():
logging.info('Using system %s' % dep.label)
else:
dep.build()
def compare_changes(repos, branch, revision):
mergeinfo_cmd = ['svn', 'mergeinfo', '--show-revs=eligible',
repos + '/trunk/CHANGES',
repos + '/' + branch + '/' + 'CHANGES']
stdout = subprocess.check_output(mergeinfo_cmd)
if stdout:
logging.warning('CHANGES has unmerged revisions: %s' %
stdout.replace("\n", " "))
_current_year = str(datetime.datetime.now().year)
_copyright_re = re.compile(r'Copyright (?:\(C\) )?(?P<year>[0-9]+)'
r' The Apache Software Foundation',
re.MULTILINE)
def check_copyright_year(repos, branch, revision):
def check_file(branch_relpath):
file_url = (repos + '/' + branch + '/'
+ branch_relpath + '@' + str(revision))
cat_cmd = ['svn', 'cat', file_url]
stdout = subprocess.check_output(cat_cmd)
m = _copyright_re.search(stdout)
if m:
year = m.group('year')
else:
year = None
if year != _current_year:
logging.warning('Copyright year in ' + branch_relpath
+ ' is not the current year')
check_file('NOTICE')
check_file('subversion/libsvn_subr/version.c')
def replace_lines(path, actions):
with open(path, 'r') as old_content:
lines = old_content.readlines()
with open(path, 'w') as new_content:
for line in lines:
for start, pattern, repl in actions:
if line.startswith(start):
line = re.sub(pattern, repl, line)
new_content.write(line)
def roll_tarballs(args):
'Create the release artifacts.'
if not args.branch:
args.branch = 'branches/%d.%d.x' % (args.version.major, args.version.minor)
branch = args.branch branch = branch.rstrip('/')
logging.info('Rolling release %s from branch %s@%d' % (args.version,
branch, args.revnum))
check_copyright_year(repos, args.branch, args.revnum)
autoconf = AutoconfDep(args.base_dir, False, args.verbose,
tool_versions[args.version.branch]['autoconf'][0],
tool_versions[args.version.branch]['autoconf'][1])
libtool = LibtoolDep(args.base_dir, False, args.verbose,
tool_versions[args.version.branch]['libtool'][0],
tool_versions[args.version.branch]['libtool'][1])
swig = SwigDep(args.base_dir, False, args.verbose,
tool_versions[args.version.branch]['swig'][0],
tool_versions[args.version.branch]['swig'][1], None)
for dep in [autoconf, libtool, swig]:
if not dep.have_usable():
raise RuntimeError('Cannot find usable %s' % dep.label)
if branch != 'trunk':
compare_changes(repos, branch, args.revnum)
if os.path.exists(get_deploydir(args.base_dir)):
raise RuntimeError('output directory \'%s\' already exists'
% get_deploydir(args.base_dir))
os.mkdir(get_deploydir(args.base_dir))
logging.info('Preparing working copy source')
shutil.rmtree(get_workdir(args.base_dir), True)
run_script(args.verbose, 'svn checkout %s %s'
% (repos + '/' + branch + '@' + str(args.revnum),
get_workdir(args.base_dir)))
exclude = ['contrib', 'notes']
if branch != 'trunk':
exclude += ['STATUS']
if args.version.minor < 7:
exclude += ['packages', 'www']
cwd = os.getcwd()
os.chdir(get_workdir(args.base_dir))
run_script(args.verbose,
'svn update --set-depth exclude %s' % " ".join(exclude))
os.chdir(cwd)
if args.patches:
majmin = '%d.%d' % (args.version.major, args.version.minor)
for name in os.listdir(args.patches):
if name.find(majmin) != -1 and name.endswith('patch'):
logging.info('Applying patch %s' % name)
run_script(args.verbose,
'''svn patch %s %s'''
% (os.path.join(args.patches, name),
get_workdir(args.base_dir)))
ver_tag, ver_numtag = args.version.get_ver_tags(args.revnum)
replacements = [('#define SVN_VER_TAG',
'".*"', ver_tag),
('#define SVN_VER_NUMTAG',
'".*"', ver_numtag),
('#define SVN_VER_REVISION',
'[0-9][0-9]*', str(args.revnum))]
if args.version.pre != 'nightly':
replacements += [('#define SVN_VER_MAJOR',
'[0-9][0-9]*', str(args.version.major)),
('#define SVN_VER_MINOR',
'[0-9][0-9]*', str(args.version.minor)),
('#define SVN_VER_PATCH',
'[0-9][0-9]*', str(args.version.patch))]
replace_lines(os.path.join(get_workdir(args.base_dir),
'subversion', 'include', 'svn_version.h'),
replacements)
exportdir = get_exportdir(args.base_dir, args.version, args.revnum)
basename = os.path.basename(exportdir)
def export(windows):
shutil.rmtree(exportdir, True)
if windows:
eol_style = "--native-eol CRLF"
else:
eol_style = "--native-eol LF"
run_script(args.verbose, "svn export %s %s %s"
% (eol_style, get_workdir(args.base_dir), exportdir))
def transform_sql():
for root, dirs, files in os.walk(exportdir):
for fname in files:
if fname.endswith('.sql'):
run_script(args.verbose,
'python build/transform_sql.py %s/%s %s/%s'
% (root, fname, root, fname[:-4] + '.h'))
def clean_autom4te():
for root, dirs, files in os.walk(get_workdir(args.base_dir)):
for dname in dirs:
if dname.startswith('autom4te') and dname.endswith('.cache'):
shutil.rmtree(os.path.join(root, dname))
logging.info('Building Windows tarballs')
export(windows=True)
os.chdir(exportdir)
transform_sql()
run_script(args.verbose,
'%s/tools/po/po-update.sh pot' % get_workdir(args.base_dir))
os.chdir(cwd)
clean_autom4te() os.chdir(get_tempdir(args.base_dir))
run_script(args.verbose,
'zip -q -r %s %s' % (basename + '.zip', basename))
os.chdir(cwd)
logging.info('Building Unix tarballs')
export(windows=False)
os.chdir(exportdir)
transform_sql()
run_script(args.verbose,
'''tools/po/po-update.sh pot
./autogen.sh --release''',
hide_stderr=True) os.chdir(cwd)
clean_autom4te()
os.chdir(get_tempdir(args.base_dir))
run_script(args.verbose,
'''pax -x ustar -w -f %s %s
bzip2 -9fk %s
gzip -9nf %s'''
% (basename + '.tar', basename,
basename + '.tar',
basename + '.tar'))
os.chdir(cwd)
logging.info('Moving artifacts and calculating checksums')
for e in extns:
filename = basename + '.' + e
filepath = os.path.join(get_tempdir(args.base_dir), filename)
shutil.move(filepath, get_deploydir(args.base_dir))
filepath = os.path.join(get_deploydir(args.base_dir), filename)
m = hashlib.sha1()
m.update(open(filepath, 'r').read())
open(filepath + '.sha1', 'w').write(m.hexdigest())
m = hashlib.sha512()
m.update(open(filepath, 'r').read())
open(filepath + '.sha512', 'w').write(m.hexdigest())
if args.version.pre != 'nightly':
shutil.copy(os.path.join(get_workdir(args.base_dir),
'subversion', 'include', 'svn_version.h'),
os.path.join(get_deploydir(args.base_dir),
'svn_version.h.dist-%s' % str(args.version)))
def sign_candidates(args):
'Sign candidate artifacts in the dist development directory.'
def sign_file(filename):
asc_file = open(filename + '.asc', 'a')
logging.info("Signing %s" % filename)
proc = subprocess.check_call(['gpg', '-ba', '-o', '-', filename],
stdout=asc_file)
asc_file.close()
target = get_target(args)
for e in extns:
filename = os.path.join(target, 'subversion-%s.%s' % (args.version, e))
sign_file(filename)
if args.version.major >= 1 and args.version.minor <= 6:
filename = os.path.join(target,
'subversion-deps-%s.%s' % (args.version, e))
sign_file(filename)
def post_candidates(args):
'Post candidate artifacts to the dist development directory.'
target = get_target(args)
logging.info('Importing tarballs to %s' % dist_dev_url)
ver = str(args.version)
svn_cmd = ['svn', 'import', '-m',
'Add Subversion %s candidate release artifacts' % ver,
'--auto-props', '--config-option',
'config:auto-props:*.asc=svn:eol-style=native;svn:mime-type=text/plain',
target, dist_dev_url]
if (args.username):
svn_cmd += ['--username', args.username]
subprocess.check_call(svn_cmd)
def create_tag(args):
'Create tag in the repository'
target = get_target(args)
logging.info('Creating tag for %s' % str(args.version))
if not args.branch:
args.branch = 'branches/%d.%d.x' % (args.version.major, args.version.minor)
branch = secure_repos + '/' + args.branch.rstrip('/')
tag = secure_repos + '/tags/' + str(args.version)
svnmucc_cmd = ['svnmucc', '-m',
'Tagging release ' + str(args.version)]
if (args.username):
svnmucc_cmd += ['--username', args.username]
svnmucc_cmd += ['cp', str(args.revnum), branch, tag]
svnmucc_cmd += ['put', os.path.join(target, 'svn_version.h.dist' + '-' +
str(args.version)),
tag + '/subversion/include/svn_version.h']
try:
subprocess.check_call(svnmucc_cmd)
except subprocess.CalledProcessError:
if args.version.is_prerelease():
logging.error("Do you need to pass --branch=trunk?")
raise
if not args.version.is_prerelease():
logging.info('Bumping revisions on the branch')
def replace_in_place(fd, startofline, flat, spare):
"""In file object FD, replace FLAT with SPARE in the first line
starting with STARTOFLINE."""
fd.seek(0, os.SEEK_SET)
lines = fd.readlines()
for i, line in enumerate(lines):
if line.startswith(startofline):
lines[i] = line.replace(flat, spare)
break
else:
raise RuntimeError('Definition of %r not found' % startofline)
fd.seek(0, os.SEEK_SET)
fd.writelines(lines)
fd.truncate()
new_version = Version('%d.%d.%d' %
(args.version.major, args.version.minor,
args.version.patch + 1))
def file_object_for(relpath):
fd = tempfile.NamedTemporaryFile()
url = branch + '/' + relpath
fd.url = url
subprocess.check_call(['svn', 'cat', '%s@%d' % (url, args.revnum)],
stdout=fd)
return fd
svn_version_h = file_object_for('subversion/include/svn_version.h')
replace_in_place(svn_version_h, '#define SVN_VER_PATCH ',
str(args.version.patch), str(new_version.patch))
STATUS = file_object_for('STATUS')
replace_in_place(STATUS, 'Status of ',
str(args.version), str(new_version))
svn_version_h.seek(0, os.SEEK_SET)
STATUS.seek(0, os.SEEK_SET)
subprocess.check_call(['svnmucc', '-r', str(args.revnum),
'-m', 'Post-release housekeeping: '
'bump the %s branch to %s.'
% (branch.split('/')[-1], str(new_version)),
'put', svn_version_h.name, svn_version_h.url,
'put', STATUS.name, STATUS.url,
])
del svn_version_h
del STATUS
def clean_dist(args):
'Clean the distribution directory of all but the most recent artifacts.'
stdout = subprocess.check_output(['svn', 'list', dist_release_url])
def minor(version):
"""Return the minor release line of the parameter, which must be
a Version object."""
return (version.major, version.minor)
filenames = stdout.split('\n')
filenames = filter(lambda x: x.startswith('subversion-'), filenames)
versions = set(map(Version, filenames))
minor_lines = set(map(minor, versions))
to_keep = set()
for recent_line in sorted(minor_lines, reverse=True)[:3]:
to_keep.add(max(
x for x in versions
if minor(x) == recent_line
))
for i in sorted(to_keep):
logging.info("Saving release '%s'", i)
svnmucc_cmd = ['svnmucc', '-m', 'Remove old Subversion releases.\n' +
'They are still available at ' +
'https://archive.apache.org/dist/subversion/']
if (args.username):
svnmucc_cmd += ['--username', args.username]
for filename in filenames:
if Version(filename) not in to_keep:
logging.info("Removing %r", filename)
svnmucc_cmd += ['rm', dist_release_url + '/' + filename]
if 'rm' in svnmucc_cmd:
subprocess.check_call(svnmucc_cmd)
else:
logging.info("Nothing to remove")
def move_to_dist(args):
'Move candidate artifacts to the distribution directory.'
stdout = subprocess.check_output(['svn', 'list', dist_dev_url])
filenames = []
for entry in stdout.split('\n'):
if fnmatch.fnmatch(entry, 'subversion-%s.*' % str(args.version)):
filenames.append(entry)
svnmucc_cmd = ['svnmucc', '-m',
'Publish Subversion-%s.' % str(args.version)]
if (args.username):
svnmucc_cmd += ['--username', args.username]
svnmucc_cmd += ['rm', dist_dev_url + '/' + 'svn_version.h.dist'
+ '-' + str(args.version)]
for filename in filenames:
svnmucc_cmd += ['mv', dist_dev_url + '/' + filename,
dist_release_url + '/' + filename]
logging.info('Moving release artifacts to %s' % dist_release_url)
subprocess.check_call(svnmucc_cmd)
def write_news(args):
'Write text for the Subversion website.'
data = { 'date' : datetime.date.today().strftime('%Y%m%d'),
'date_pres' : datetime.date.today().strftime('%Y-%m-%d'),
'major-minor' : args.version.branch,
'version' : str(args.version),
'version_base' : args.version.base,
'anchor': args.version.get_download_anchor(),
'is_recommended': ezt_bool(args.version.is_recommended()),
}
if args.version.is_prerelease():
template_filename = 'rc-news.ezt'
else:
template_filename = 'stable-news.ezt'
template = ezt.Template()
template.parse(get_tmplfile(template_filename).read())
template.generate(sys.stdout, data)
def get_sha1info(args):
'Return a list of sha1 info for the release'
target = get_target(args)
sha1s = glob.glob(os.path.join(target, 'subversion*-%s*.sha1' % args.version))
class info(object):
pass
sha1info = []
for s in sha1s:
i = info()
i.filename = os.path.basename(s)[:-5]
i.sha1 = open(s, 'r').read()
sha1info.append(i)
return sha1info
def write_announcement(args):
'Write the release announcement.'
sha1info = get_sha1info(args)
siginfo = "\n".join(get_siginfo(args, True)) + "\n"
data = { 'version' : str(args.version),
'sha1info' : sha1info,
'siginfo' : siginfo,
'major-minor' : args.version.branch,
'major-minor-patch' : args.version.base,
'anchor' : args.version.get_download_anchor(),
}
if args.version.is_prerelease():
template_filename = 'rc-release-ann.ezt'
else:
data['dot-zero'] = ezt_bool(args.version.patch == 0)
data['security'] = ezt_bool(args.security)
template_filename = 'stable-release-ann.ezt'
assert not (data['dot-zero'] and data['security'])
template = ezt.Template(compress_whitespace = False)
template.parse(get_tmplfile(template_filename).read())
template.generate(sys.stdout, data)
def write_downloads(args):
'Output the download section of the website.'
sha1info = get_sha1info(args)
data = { 'version' : str(args.version),
'fileinfo' : sha1info,
}
template = ezt.Template(compress_whitespace = False)
template.parse(get_tmplfile('download.ezt').read())
template.generate(sys.stdout, data)
key_start = '-----BEGIN PGP SIGNATURE-----'
key_end = '-----END PGP SIGNATURE-----'
PUBLIC_KEY_ALGORITHMS = {
1: (lambda keylen: str(keylen) + 'R'), }
def _make_human_readable_fingerprint(fingerprint):
return re.compile(r'(....)' * 10).sub(r'\1 \2 \3 \4 \5 \6 \7 \8 \9 \10',
fingerprint)
def get_siginfo(args, quiet=False):
'Returns a list of signatures for the release.'
try:
import gnupg
except ImportError:
import security._gnupg as gnupg
gpg = gnupg.GPG()
target = get_target(args)
good_sigs = {}
fingerprints = {}
output = []
glob_pattern = os.path.join(target, 'subversion*-%s*.asc' % args.version)
for filename in glob.glob(glob_pattern):
text = open(filename).read()
keys = text.split(key_start)
if keys[0]:
sys.stderr.write("SYNTAX ERROR: %s does not start with '%s'\n"
% (filename, key_start))
sys.exit(1)
keys = keys[1:]
if not quiet:
logging.info("Checking %d sig(s) in %s" % (len(keys), filename))
n = 0
for key in keys:
n += 1
if not key.rstrip().endswith(key_end):
sys.stderr.write("SYNTAX ERROR: Key %d in %s"
" does not end with '%s'\n"
% (n, filename, key_end))
sys.exit(1)
fd, fn = tempfile.mkstemp()
os.write(fd, key_start + key)
os.close(fd)
verified = gpg.verify_file(open(fn, 'rb'), filename[:-4])
os.unlink(fn)
if verified.valid:
good_sigs[verified.fingerprint] = True
else:
sys.stderr.write("BAD SIGNATURE: Key %d in %s\n"
% (n, filename))
if verified.key_id:
sys.stderr.write(" key id: %s\n" % verified.key_id)
sys.exit(1)
for id in good_sigs.keys():
gpg_output = subprocess.check_output(
['gpg', '--fixed-list-mode', '--with-colons', '--fingerprint', id],
stderr=subprocess.STDOUT,
)
gpg_output = gpg_output.splitlines()
for line in gpg_output:
parts = line.split(':')
if parts[0] == 'pub':
keylen = int(parts[2])
keytype = int(parts[3])
formatter = PUBLIC_KEY_ALGORITHMS[keytype]
long_key_id = parts[4]
length_and_type = formatter(keylen) + '/' + long_key_id
del keylen, keytype, formatter, long_key_id
break
else:
raise RuntimeError("Failed to determine LONG_KEY_ID")
for line in gpg_output:
parts = line.split(':')
if parts[0] == 'fpr':
fingerprint = parts[9]
break
else:
raise RuntimeError("Failed to determine FINGERPRINT")
for line in gpg_output:
parts = line.split(':')
if parts[0] == 'uid':
name = parts[9].split(' <')[0]
break
else:
raise RuntimeError("Failed to determine NAME")
format_expandos = dict(
name=name,
length_and_type=length_and_type,
fingerprint=_make_human_readable_fingerprint(fingerprint),
)
del name, length_and_type, fingerprint
line = " {name} [{length_and_type}] with fingerprint:"
output.append( line.format(**format_expandos) )
line = " {fingerprint}"
output.append( line.format(**format_expandos) )
return output
def check_sigs(args):
'Check the signatures for the release.'
output = get_siginfo(args)
for line in output:
print(line)
def get_keys(args):
'Import the LDAP-based KEYS file to gpg'
with tempfile.SpooledTemporaryFile() as fd:
fd.write(urllib2.urlopen(KEYS).read())
fd.flush()
fd.seek(0)
subprocess.check_call(['gpg', '--import'], stdin=fd)
def main():
'Parse arguments, and drive the appropriate subcommand.'
parser = argparse.ArgumentParser(
description='Create an Apache Subversion release.')
parser.add_argument('--clean', action='store_true', default=False,
help='Remove any directories previously created by %(prog)s')
parser.add_argument('--verbose', action='store_true', default=False,
help='Increase output verbosity')
parser.add_argument('--base-dir', default=os.getcwd(),
help='''The directory in which to create needed files and
folders. The default is the current working
directory.''')
subparsers = parser.add_subparsers(title='subcommands')
subparser = subparsers.add_parser('build-env',
help='''Download release prerequisistes, including autoconf,
libtool, and swig.''')
subparser.set_defaults(func=build_env)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('--sf-mirror', default='softlayer',
help='''The mirror to use for downloading files from
SourceForge. If in the EU, you may want to use
'kent' for this value.''')
subparser.add_argument('--use-existing', action='store_true', default=False,
help='''Attempt to use existing build dependencies before
downloading and building a private set.''')
subparser = subparsers.add_parser('roll',
help='''Create the release artifacts.''')
subparser.set_defaults(func=roll_tarballs)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')),
help='''The revision number to base the release on.''')
subparser.add_argument('--branch',
help='''The branch to base the release on,
relative to ^/subversion/.''')
subparser.add_argument('--patches',
help='''The path to the directory containing patches.''')
subparser = subparsers.add_parser('sign-candidates',
help='''Sign the release artifacts.''')
subparser.set_defaults(func=sign_candidates)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('--target',
help='''The full path to the directory containing
release artifacts.''')
subparser = subparsers.add_parser('post-candidates',
help='''Commit candidates to the release development area
of the dist.apache.org repository.''')
subparser.set_defaults(func=post_candidates)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('--username',
help='''Username for ''' + dist_repos + '''.''')
subparser.add_argument('--target',
help='''The full path to the directory containing
release artifacts.''')
subparser = subparsers.add_parser('create-tag',
help='''Create the release tag.''')
subparser.set_defaults(func=create_tag)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')),
help='''The revision number to base the release on.''')
subparser.add_argument('--branch',
help='''The branch to base the release on,
relative to ^/subversion/.''')
subparser.add_argument('--username',
help='''Username for ''' + secure_repos + '''.''')
subparser.add_argument('--target',
help='''The full path to the directory containing
release artifacts.''')
subparser = subparsers.add_parser('clean-dist',
help='''Clean the distribution directory (and mirrors) of
all but the most recent MAJOR.MINOR release.''')
subparser.set_defaults(func=clean_dist)
subparser.add_argument('--dist-dir',
help='''The directory to clean.''')
subparser.add_argument('--username',
help='''Username for ''' + dist_repos + '''.''')
subparser = subparsers.add_parser('move-to-dist',
help='''Move candiates and signatures from the temporary
release dev location to the permanent distribution
directory.''')
subparser.set_defaults(func=move_to_dist)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('--username',
help='''Username for ''' + dist_repos + '''.''')
subparser = subparsers.add_parser('write-news',
help='''Output to stdout template text for use in the news
section of the Subversion website.''')
subparser.set_defaults(func=write_news)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser = subparsers.add_parser('write-announcement',
help='''Output to stdout template text for the emailed
release announcement.''')
subparser.set_defaults(func=write_announcement)
subparser.add_argument('--security', action='store_true', default=False,
help='''The release being announced includes security
fixes.''')
subparser.add_argument('--target',
help='''The full path to the directory containing
release artifacts.''')
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser = subparsers.add_parser('write-downloads',
help='''Output to stdout template text for the download
table for subversion.apache.org''')
subparser.set_defaults(func=write_downloads)
subparser.add_argument('--target',
help='''The full path to the directory containing
release artifacts.''')
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser = subparsers.add_parser('check-sigs',
help='''Output to stdout the signatures collected for this
release''')
subparser.set_defaults(func=check_sigs)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('--target',
help='''The full path to the directory containing
release artifacts.''')
subparser = subparsers.add_parser('get-keys',
help='''Import committers' public keys to ~/.gpg/''')
subparser.set_defaults(func=get_keys)
subparser = subparsers.add_parser('clean',
help='''The same as the '--clean' switch, but as a
separate subcommand.''')
subparser.set_defaults(func=cleanup)
args = parser.parse_args()
if args.clean:
cleanup(args)
logger = logging.getLogger()
if args.verbose:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \
+ os.environ['PATH']
os.environ['TZ'] = 'UTC'
args.func(args)
if __name__ == '__main__':
main()