diff options
Diffstat (limited to 'python/mozbuild/mozpack/unify.py')
| -rw-r--r-- | python/mozbuild/mozpack/unify.py | 231 |
1 files changed, 231 insertions, 0 deletions
diff --git a/python/mozbuild/mozpack/unify.py b/python/mozbuild/mozpack/unify.py new file mode 100644 index 0000000000..3c8a8d6057 --- /dev/null +++ b/python/mozbuild/mozpack/unify.py @@ -0,0 +1,231 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from mozpack.files import ( + BaseFinder, + JarFinder, + ExecutableFile, + BaseFile, + GeneratedFile, +) +from mozpack.executables import ( + MACHO_SIGNATURES, +) +from mozpack.mozjar import JarReader +from mozpack.errors import errors +from tempfile import mkstemp +import mozpack.path as mozpath +import struct +import os +import re +import subprocess +import buildconfig +from collections import OrderedDict + +# Regular expressions for unifying install.rdf +FIND_TARGET_PLATFORM = re.compile(r""" + <(?P<ns>[-._0-9A-Za-z]+:)?targetPlatform> # The targetPlatform tag, with any namespace + (?P<platform>[^<]*) # The actual platform value + </(?P=ns)?targetPlatform> # The closing tag + """, re.X) +FIND_TARGET_PLATFORM_ATTR = re.compile(r""" + (?P<tag><(?:[-._0-9A-Za-z]+:)?Description) # The opening part of the <Description> tag + (?P<attrs>[^>]*?)\s+ # The initial attributes + (?P<ns>[-._0-9A-Za-z]+:)?targetPlatform= # The targetPlatform attribute, with any namespace + [\'"](?P<platform>[^\'"]+)[\'"] # The actual platform value + (?P<otherattrs>[^>]*?>) # The remaining attributes and closing angle bracket + """, re.X) + +def may_unify_binary(file): + ''' + Return whether the given BaseFile instance is an ExecutableFile that + may be unified. Only non-fat Mach-O binaries are to be unified. + ''' + if isinstance(file, ExecutableFile): + signature = file.open().read(4) + if len(signature) < 4: + return False + signature = struct.unpack('>L', signature)[0] + if signature in MACHO_SIGNATURES: + return True + return False + + +class UnifiedExecutableFile(BaseFile): + ''' + File class for executable and library files that to be unified with 'lipo'. + ''' + def __init__(self, executable1, executable2): + ''' + Initialize a UnifiedExecutableFile with a pair of ExecutableFiles to + be unified. They are expected to be non-fat Mach-O executables. + ''' + assert isinstance(executable1, ExecutableFile) + assert isinstance(executable2, ExecutableFile) + self._executables = (executable1, executable2) + + def copy(self, dest, skip_if_older=True): + ''' + Create a fat executable from the two Mach-O executable given when + creating the instance. + skip_if_older is ignored. + ''' + assert isinstance(dest, basestring) + tmpfiles = [] + try: + for e in self._executables: + fd, f = mkstemp() + os.close(fd) + tmpfiles.append(f) + e.copy(f, skip_if_older=False) + lipo = buildconfig.substs.get('LIPO') or 'lipo' + subprocess.call([lipo, '-create'] + tmpfiles + ['-output', dest]) + finally: + for f in tmpfiles: + os.unlink(f) + + +class UnifiedFinder(BaseFinder): + ''' + Helper to get unified BaseFile instances from two distinct trees on the + file system. + ''' + def __init__(self, finder1, finder2, sorted=[], **kargs): + ''' + Initialize a UnifiedFinder. finder1 and finder2 are BaseFinder + instances from which files are picked. UnifiedFinder.find() will act as + FileFinder.find() but will error out when matches can only be found in + one of the two trees and not the other. It will also error out if + matches can be found on both ends but their contents are not identical. + + The sorted argument gives a list of mozpath.match patterns. File + paths matching one of these patterns will have their contents compared + with their lines sorted. + ''' + assert isinstance(finder1, BaseFinder) + assert isinstance(finder2, BaseFinder) + self._finder1 = finder1 + self._finder2 = finder2 + self._sorted = sorted + BaseFinder.__init__(self, finder1.base, **kargs) + + def _find(self, path): + ''' + UnifiedFinder.find() implementation. + ''' + files1 = OrderedDict() + for p, f in self._finder1.find(path): + files1[p] = f + files2 = set() + for p, f in self._finder2.find(path): + files2.add(p) + if p in files1: + if may_unify_binary(files1[p]) and \ + may_unify_binary(f): + yield p, UnifiedExecutableFile(files1[p], f) + else: + err = errors.count + unified = self.unify_file(p, files1[p], f) + if unified: + yield p, unified + elif err == errors.count: + self._report_difference(p, files1[p], f) + else: + errors.error('File missing in %s: %s' % + (self._finder1.base, p)) + for p in [p for p in files1 if not p in files2]: + errors.error('File missing in %s: %s' % (self._finder2.base, p)) + + def _report_difference(self, path, file1, file2): + ''' + Report differences between files in both trees. + ''' + errors.error("Can't unify %s: file differs between %s and %s" % + (path, self._finder1.base, self._finder2.base)) + if not isinstance(file1, ExecutableFile) and \ + not isinstance(file2, ExecutableFile): + from difflib import unified_diff + for line in unified_diff(file1.open().readlines(), + file2.open().readlines(), + os.path.join(self._finder1.base, path), + os.path.join(self._finder2.base, path)): + errors.out.write(line) + + def unify_file(self, path, file1, file2): + ''' + Given two BaseFiles and the path they were found at, check whether + their content match and return the first BaseFile if they do. + ''' + content1 = file1.open().readlines() + content2 = file2.open().readlines() + if content1 == content2: + return file1 + for pattern in self._sorted: + if mozpath.match(path, pattern): + if sorted(content1) == sorted(content2): + return file1 + break + return None + + +class UnifiedBuildFinder(UnifiedFinder): + ''' + Specialized UnifiedFinder for Mozilla applications packaging. It allows + "*.manifest" files to differ in their order, and unifies "buildconfig.html" + files by merging their content. + ''' + def __init__(self, finder1, finder2, **kargs): + UnifiedFinder.__init__(self, finder1, finder2, + sorted=['**/*.manifest'], **kargs) + + def unify_file(self, path, file1, file2): + ''' + Unify files taking Mozilla application special cases into account. + Otherwise defer to UnifiedFinder.unify_file. + ''' + basename = mozpath.basename(path) + if basename == 'buildconfig.html': + content1 = file1.open().readlines() + content2 = file2.open().readlines() + # Copy everything from the first file up to the end of its <body>, + # insert a <hr> between the two files and copy the second file's + # content beginning after its leading <h1>. + return GeneratedFile(''.join( + content1[:content1.index('</body>\n')] + + ['<hr> </hr>\n'] + + content2[content2.index('<h1>about:buildconfig</h1>\n') + 1:] + )) + elif basename == 'install.rdf': + # install.rdf files often have em:targetPlatform (either as + # attribute or as tag) that will differ between platforms. The + # unified install.rdf should contain both em:targetPlatforms if + # they exist, or strip them if only one file has a target platform. + content1, content2 = ( + FIND_TARGET_PLATFORM_ATTR.sub(lambda m: \ + m.group('tag') + m.group('attrs') + m.group('otherattrs') + + '<%stargetPlatform>%s</%stargetPlatform>' % \ + (m.group('ns') or "", m.group('platform'), m.group('ns') or ""), + f.open().read() + ) for f in (file1, file2) + ) + + platform2 = FIND_TARGET_PLATFORM.search(content2) + return GeneratedFile(FIND_TARGET_PLATFORM.sub( + lambda m: m.group(0) + platform2.group(0) if platform2 else '', + content1 + )) + elif path.endswith('.xpi'): + finder1 = JarFinder(os.path.join(self._finder1.base, path), + JarReader(fileobj=file1.open())) + finder2 = JarFinder(os.path.join(self._finder2.base, path), + JarReader(fileobj=file2.open())) + unifier = UnifiedFinder(finder1, finder2, sorted=self._sorted) + err = errors.count + all(unifier.find('')) + if err == errors.count: + return file1 + return None + return UnifiedFinder.unify_file(self, path, file1, file2) |