diff options
20 files changed, 276 insertions, 118 deletions
diff --git a/config/external/moz.build b/config/external/moz.build index dc094b6af1..47db9e1526 100644 --- a/config/external/moz.build +++ b/config/external/moz.build @@ -54,7 +54,7 @@ external_dirs += [ 'media/psshparser' ] -if CONFIG['MOZ_LINKER']: +if CONFIG['MOZ_UPDATER']: external_dirs += ['modules/xz-embedded'] DIRS += ['../../' + i for i in external_dirs] diff --git a/modules/libmar/src/mar.h b/modules/libmar/src/mar.h index 776daf648d..e9c8d555e4 100644 --- a/modules/libmar/src/mar.h +++ b/modules/libmar/src/mar.h @@ -109,7 +109,7 @@ int mar_enum_items(MarFile *mar, MarItemCallback callback, void *data); * @return The number of bytes written or a negative value if an * error occurs. */ -int mar_read(MarFile *mar, const MarItem *item, int offset, char *buf, +int mar_read(MarFile *mar, const MarItem *item, int offset, uint8_t *buf, int bufsize); /** diff --git a/modules/libmar/src/mar_extract.c b/modules/libmar/src/mar_extract.c index ec1cd6c534..969697ee29 100644 --- a/modules/libmar/src/mar_extract.c +++ b/modules/libmar/src/mar_extract.c @@ -37,7 +37,7 @@ static int mar_ensure_parent_dir(const char *path) static int mar_test_callback(MarFile *mar, const MarItem *item, void *unused) { FILE *fp; - char buf[BLOCKSIZE]; + uint8_t buf[BLOCKSIZE]; int fd, len, offset = 0; if (mar_ensure_parent_dir(item->name)) diff --git a/modules/libmar/src/mar_read.c b/modules/libmar/src/mar_read.c index 241d0c08e4..89b9da27d6 100644 --- a/modules/libmar/src/mar_read.c +++ b/modules/libmar/src/mar_read.c @@ -530,7 +530,7 @@ int mar_enum_items(MarFile *mar, MarItemCallback callback, void *closure) { return 0; } -int mar_read(MarFile *mar, const MarItem *item, int offset, char *buf, +int mar_read(MarFile *mar, const MarItem *item, int offset, uint8_t *buf, int bufsize) { int nr; diff --git a/modules/xz-embedded/moz.build b/modules/xz-embedded/moz.build index fa1e5fc468..9806dca1ea 100644 --- a/modules/xz-embedded/moz.build +++ b/modules/xz-embedded/moz.build @@ -29,4 +29,7 @@ elif '86' in CONFIG['TARGET_CPU']: DEFINES['XZ_USE_CRC64'] = 1 +if CONFIG['OS_ARCH'] == 'WINNT': + USE_STATIC_LIBS = True + Library('xz-embedded') diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index a2983b48c2..4f58a2a33d 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -1140,8 +1140,13 @@ class Mar(MachCommandBase): @Command('mar', category='post-build', description='Create the mar file for the built product for distribution.') - def mar(self): - return self._run_make(directory="./tools/update-packaging/", target='', ensure_exit_code=False) + @CommandArgument('--bz2', action='store_true', + help='Compress the mar package with old-style bz2 instead of xz') + def mar(self, bz2): + if bz2: + return self._run_make(directory="./tools/update-packaging/", target='mar-package-bz2', ensure_exit_code=False) + else: + return self._run_make(directory="./tools/update-packaging/", target='mar-package', ensure_exit_code=False) @CommandProvider class Install(MachCommandBase): diff --git a/toolkit/mozapps/installer/package-name.mk b/toolkit/mozapps/installer/package-name.mk index 680590b3c5..548a93b47f 100644 --- a/toolkit/mozapps/installer/package-name.mk +++ b/toolkit/mozapps/installer/package-name.mk @@ -80,6 +80,7 @@ PKG_UPDATE_BASENAME = $(PKG_BASENAME) CHECKSUMS_FILE_BASENAME = $(PKG_BASENAME) MOZ_INFO_BASENAME = $(PKG_BASENAME) PKG_UPDATE_PATH = $(PKG_PATH) +BARE_MAR = $(PKG_UPDATE_PATH)$(PKG_UPDATE_BASENAME).mar COMPLETE_MAR = $(PKG_UPDATE_PATH)$(PKG_UPDATE_BASENAME).complete.mar # PARTIAL_MAR needs to be processed by $(wildcard) before you use it. PARTIAL_MAR = $(PKG_UPDATE_PATH)$(PKG_UPDATE_BASENAME).partial.*.mar diff --git a/toolkit/mozapps/update/common/errors.h b/toolkit/mozapps/update/common/errors.h index 846118d55d..de1782b8e5 100644 --- a/toolkit/mozapps/update/common/errors.h +++ b/toolkit/mozapps/update/common/errors.h @@ -46,7 +46,7 @@ #define WRITE_ERROR_ACCESS_DENIED 35 // #define WRITE_ERROR_SHARING_VIOLATION 36 // Replaced with errors 46-48 #define WRITE_ERROR_CALLBACK_APP 37 -#define UNEXPECTED_BZIP_ERROR 39 +#define UNEXPECTED_XZ_ERROR 39 #define UNEXPECTED_MAR_ERROR 40 #define UNEXPECTED_BSPATCH_ERROR 41 #define UNEXPECTED_FILE_OPERATION_ERROR 42 diff --git a/toolkit/mozapps/update/updater/archivereader.cpp b/toolkit/mozapps/update/updater/archivereader.cpp index 90cf45c3db..72debbe82e 100644 --- a/toolkit/mozapps/update/updater/archivereader.cpp +++ b/toolkit/mozapps/update/updater/archivereader.cpp @@ -1,5 +1,4 @@ /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -15,6 +14,9 @@ #include "updatehelper.h" #endif +#define XZ_USE_CRC64 +#include "xz.h" + // These are generated at compile time based on the DER file for the channel // being used #ifdef MOZ_VERIFY_MAR_SIGNATURE @@ -36,10 +38,10 @@ # include <io.h> #endif -static int inbuf_size = 262144; -static int outbuf_size = 262144; -static char *inbuf = nullptr; -static char *outbuf = nullptr; +static size_t inbuf_size = 262144; +static size_t outbuf_size = 262144; +static uint8_t *inbuf = nullptr; +static uint8_t *outbuf = nullptr; /** * Performs a verification on the opened MAR file with the passed in @@ -182,22 +184,22 @@ ArchiveReader::Open(const NS_tchar *path) Close(); if (!inbuf) { - inbuf = (char *)malloc(inbuf_size); + inbuf = (uint8_t *)malloc(inbuf_size); if (!inbuf) { // Try again with a smaller buffer. inbuf_size = 1024; - inbuf = (char *)malloc(inbuf_size); + inbuf = (uint8_t *)malloc(inbuf_size); if (!inbuf) return ARCHIVE_READER_MEM_ERROR; } } if (!outbuf) { - outbuf = (char *)malloc(outbuf_size); + outbuf = (uint8_t *)malloc(outbuf_size); if (!outbuf) { // Try again with a smaller buffer. outbuf_size = 1024; - outbuf = (char *)malloc(outbuf_size); + outbuf = (uint8_t *)malloc(outbuf_size); if (!outbuf) return ARCHIVE_READER_MEM_ERROR; } @@ -211,6 +213,9 @@ ArchiveReader::Open(const NS_tchar *path) if (!mArchive) return READ_ERROR; + xz_crc32_init(); + xz_crc64_init(); + return OK; } @@ -273,12 +278,21 @@ ArchiveReader::ExtractItemToStream(const MarItem *item, FILE *fp) { /* decompress the data chunk by chunk */ - bz_stream strm; - int offset, inlen, outlen, ret = OK; + int offset, inlen, ret = OK; + struct xz_buf strm = { 0 }; + enum xz_ret xz_rv = XZ_OK; + + struct xz_dec * dec = xz_dec_init(XZ_DYNALLOC, 64 * 1024 * 1024); + if (!dec) { + return UNEXPECTED_XZ_ERROR; + } - memset(&strm, 0, sizeof(strm)); - if (BZ2_bzDecompressInit(&strm, 0, 0) != BZ_OK) - return UNEXPECTED_BZIP_ERROR; + strm.in = inbuf; + strm.in_pos = 0; + strm.in_size = 0; + strm.out = outbuf; + strm.out_pos = 0; + strm.out_size = outbuf_size; offset = 0; for (;;) { @@ -287,38 +301,50 @@ ArchiveReader::ExtractItemToStream(const MarItem *item, FILE *fp) break; } - if (offset < (int) item->length && strm.avail_in == 0) { + if (offset < (int) item->length && strm.in_pos == strm.in_size) { inlen = mar_read(mArchive, item, offset, inbuf, inbuf_size); - if (inlen <= 0) - return READ_ERROR; + if (inlen <= 0) { + ret = READ_ERROR; + break; + } offset += inlen; - strm.next_in = inbuf; - strm.avail_in = inlen; + strm.in_size = inlen; + strm.in_pos = 0; } - strm.next_out = outbuf; - strm.avail_out = outbuf_size; + xz_rv = xz_dec_run(dec, &strm); - ret = BZ2_bzDecompress(&strm); - if (ret != BZ_OK && ret != BZ_STREAM_END) { - ret = UNEXPECTED_BZIP_ERROR; - break; - } - - outlen = outbuf_size - strm.avail_out; - if (outlen) { - if (fwrite(outbuf, outlen, 1, fp) != 1) { + if (strm.out_pos == outbuf_size) { + if (fwrite(outbuf, 1, strm.out_pos, fp) != strm.out_pos) { ret = WRITE_ERROR_EXTRACT; break; } + + strm.out_pos = 0; + } + + if (xz_rv == XZ_OK) { + // There is still more data to decompress. + continue; } - if (ret == BZ_STREAM_END) { - ret = OK; + // The return value of xz_dec_run is not XZ_OK and if it isn't XZ_STREAM_END + // an error has occured. + if (xz_rv != XZ_STREAM_END) { + ret = UNEXPECTED_XZ_ERROR; break; } + + // Write out the remainder of the decompressed data. In the case of + // strm.out_pos == 0 this is needed to create empty files included in the + // mar file. + if (fwrite(outbuf, 1, strm.out_pos, fp) != strm.out_pos) { + ret = WRITE_ERROR_EXTRACT; + } + + break; } - BZ2_bzDecompressEnd(&strm); + xz_dec_end(dec); return ret; } diff --git a/toolkit/mozapps/update/updater/updater-common.build b/toolkit/mozapps/update/updater/updater-common.build index a46c1388cc..1ace8fcc71 100644 --- a/toolkit/mozapps/update/updater/updater-common.build +++ b/toolkit/mozapps/update/updater/updater-common.build @@ -57,6 +57,7 @@ else: USE_LIBS += [ 'mar', + 'xz-embedded', ] if CONFIG['MOZ_SYSTEM_BZ2']: diff --git a/tools/update-packaging/Makefile.in b/tools/update-packaging/Makefile.in index 4788a55752..a751ae3ce5 100644 --- a/tools/update-packaging/Makefile.in +++ b/tools/update-packaging/Makefile.in @@ -31,7 +31,7 @@ MAR_BIN = $(DIST)/host/bin/mar$(HOST_BIN_SUFFIX) MBSDIFF_BIN = $(DIST)/host/bin/mbsdiff$(HOST_BIN_SUFFIX) OVERRIDE_DEFAULT_GOAL := full-update -full-update:: complete-patch +full-update:: mar-package ifeq ($(OS_TARGET), WINNT) ifdef AB_CD @@ -70,3 +70,23 @@ partial-patch:: $(dir-stage) ifdef MOZ_SIGN_CMD $(MOZ_SIGN_CMD) -f mar '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar' endif + +mar-package: $(call mkdir_deps,$(ABS_DIST)/$(PKG_STAGE_DIR)) + @echo 'Creating update mar (xz compressed)...' + MAR=$(MAR_BIN) \ + $(srcdir)/make_full_update.sh \ + '$(DIST)/$(BARE_MAR)' \ + '$(PACKAGE_DIR)' +ifdef MOZ_SIGN_CMD + $(MOZ_SIGN_CMD) -f mar '$(DIST)/$(BARE_MAR)' +endif + +mar-package-bz2: $(call mkdir_deps,$(ABS_DIST)/$(PKG_STAGE_DIR)) + @echo 'Creating update mar (bz2 compressed)...' + MAR_OLD_FORMAT=1 MAR=$(MAR_BIN) \ + $(srcdir)/make_full_update.sh \ + '$(DIST)/$(COMPLETE_MAR)' \ + '$(PACKAGE_DIR)' +ifdef MOZ_SIGN_CMD + $(MOZ_SIGN_CMD) -f mar '$(DIST)/$(COMPLETE_MAR)' +endif
\ No newline at end of file diff --git a/tools/update-packaging/common.sh b/tools/update-packaging/common.sh index 24f1ca6eb4..3df2cc4768 100755 --- a/tools/update-packaging/common.sh +++ b/tools/update-packaging/common.sh @@ -9,10 +9,15 @@ # # ----------------------------------------------------------------------------- -# By default just assume that these tools exist on our path +# By default just assume that these tools exist in our path MAR=${MAR:-mar} -BZIP2=${BZIP2:-bzip2} MBSDIFF=${MBSDIFF:-mbsdiff} +if [[ -z "${MAR_OLD_FORMAT}" ]]; then + XZ=${XZ:-xz} +else + MAR_OLD_FORMAT=1 + BZIP2=${BZIP2:-bzip2} +fi # ----------------------------------------------------------------------------- # Helper routines diff --git a/tools/update-packaging/make_full_update.sh b/tools/update-packaging/make_full_update.sh index ba1404474f..c480f8ee66 100755 --- a/tools/update-packaging/make_full_update.sh +++ b/tools/update-packaging/make_full_update.sh @@ -93,7 +93,11 @@ for ((i=0; $i<$num_files; i=$i+1)); do dir=$(dirname "$f") mkdir -p "$workdir/$dir" - $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" + if [[ -n $MAR_OLD_FORMAT ]]; then + $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" + else + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f" + fi copy_perm "$targetdir/$f" "$workdir/$f" targetfiles="$targetfiles \"$f\"" @@ -104,8 +108,13 @@ notice "" notice "Adding file and directory remove instructions from file 'removed-files'" append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3" -$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" -$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" +if [[ -n $MAR_OLD_FORMAT ]]; then + $BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" + $BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" +else + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2" + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" +fi eval "$MAR -C \"$workdir\" -c output.mar $targetfiles" mv -f "$workdir/output.mar" "$archive" diff --git a/tools/update-packaging/make_incremental_update.sh b/tools/update-packaging/make_incremental_update.sh index 90372c017e..af3af854fd 100755 --- a/tools/update-packaging/make_incremental_update.sh +++ b/tools/update-packaging/make_incremental_update.sh @@ -169,7 +169,11 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do if check_for_add_if_not_update "$f"; then # The full workdir may not exist yet, so create it if necessary. mkdir -p `dirname "$workdir/$f"` - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + if [[ -n $MAR_OLD_FORMAT ]]; then + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + else + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + fi copy_perm "$newdir/$f" "$workdir/$f" make_add_if_not_instruction "$f" "$updatemanifestv3" archivefiles="$archivefiles \"$f\"" @@ -179,7 +183,11 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do if check_for_forced_update "$requested_forced_updates" "$f"; then # The full workdir may not exist yet, so create it if necessary. mkdir -p `dirname "$workdir/$f"` - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + if [[ -n $MAR_OLD_FORMAT ]]; then + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + else + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + fi copy_perm "$newdir/$f" "$workdir/$f" make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1 archivefiles="$archivefiles \"$f\"" @@ -207,21 +215,44 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do # if service is not enabled then default to old behavior if [ -z "$MBSDIFF_HOOK" ]; then $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" - $BZIP2 -z9 "$workdir/$f.patch" + if [[ -n $MAR_OLD_FORMAT ]]; then + $BZIP2 -z9 "$workdir/$f.patch" + else + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch" + fi else # if service enabled then check patch existence for retrieval - if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then - notice "file \"$f\" found in funsize, diffing skipped" + if [[ -n $MAR_OLD_FORMAT ]]; then + if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then + notice "file \"$f\" found in funsize, diffing skipped" + else + # if not found already - compute it and cache it for future use + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $BZIP2 -z9 "$workdir/$f.patch" + $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2" + fi else - # if not found already - compute it and cache it for future use - $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" - $BZIP2 -z9 "$workdir/$f.patch" - $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2" + if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then + notice "file \"$f\" found in funsize, diffing skipped" + else + # if not found already - compute it and cache it for future use + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch" + $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz" + fi fi fi - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + if [[ -n $MAR_OLD_FORMAT ]]; then + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + else + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + fi copy_perm "$newdir/$f" "$workdir/$f" - patchfile="$workdir/$f.patch.bz2" + if [[ -n $MAR_OLD_FORMAT ]]; then + patchfile="$workdir/$f.patch.bz2" + else + patchfile="$workdir/$f.patch.xz" + fi patchsize=$(get_file_size "$patchfile") fullsize=$(get_file_size "$workdir/$f") @@ -262,7 +293,11 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); do dir=$(dirname "$workdir/$f") mkdir -p "$dir" - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + if [[ -n $MAR_OLD_FORMAT ]]; then + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + else + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + fi copy_perm "$newdir/$f" "$workdir/$f" if check_for_add_if_not_update "$f"; then @@ -303,8 +338,13 @@ for ((i=0; $i<$num_olddirs; i=$i+1)); do fi done -$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" -$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" +if [[ -n $MAR_OLD_FORMAT ]]; then + $BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" + $BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" +else + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2" + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" +fi mar_command="$MAR" if [[ -n $MOZ_PRODUCT_VERSION ]] diff --git a/tools/update-packaging/make_incremental_updates.py b/tools/update-packaging/make_incremental_updates.py index d346ee2df0..dc7b3a01ea 100755 --- a/tools/update-packaging/make_incremental_updates.py +++ b/tools/update-packaging/make_incremental_updates.py @@ -4,7 +4,7 @@ import os import shutil -import sha +import hashlib from os.path import join, getsize from stat import * import re @@ -12,9 +12,9 @@ import sys import getopt import time import datetime -import bz2 import string import tempfile +import io class PatchInfo: """ Represents the meta-data associated with a patch @@ -107,23 +107,23 @@ class PatchInfo: """ Create the v2 manifest file in the root of the work_dir """ manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest") manifest_file = open(manifest_file_path, "wb") - manifest_file.writelines("type \"partial\"\n") - manifest_file.writelines(string.join(self.manifestv2, '\n')) - manifest_file.writelines("\n") + manifest_file.writelines(io.BytesIO(b"type \"partial\"\n")) + manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv2).encode('ascii'))) + manifest_file.writelines(io.BytesIO(b"\n")) manifest_file.close() - bzip_file(manifest_file_path) + xz_file(manifest_file_path) self.archive_files.append('"updatev2.manifest"') """ Create the v3 manifest file in the root of the work_dir """ manifest_file_path = os.path.join(self.work_dir,"updatev3.manifest") manifest_file = open(manifest_file_path, "wb") - manifest_file.writelines("type \"partial\"\n") - manifest_file.writelines(string.join(self.manifestv3, '\n')) - manifest_file.writelines("\n") + manifest_file.writelines(io.BytesIO(b"type \"partial\"\n")) + manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv3).encode('ascii'))) + manifest_file.writelines(io.BytesIO(b"\n")) manifest_file.close() - bzip_file(manifest_file_path) + xz_file(manifest_file_path) self.archive_files.append('"updatev3.manifest"') def build_marfile_entry_hash(self, root_path): @@ -176,8 +176,8 @@ class MarFileEntry: def calc_file_sha_digest(self, filename): """ Returns sha digest of given filename""" - file_content = open(filename, 'r').read() - return sha.new(file_content).digest() + file_content = open(filename, 'rb').read() + return hashlib.sha1(file_content).digest() def sha(self): """ Returns sha digest of file repreesnted by this _marfile_entry""" @@ -188,7 +188,7 @@ class MarFileEntry: def exec_shell_cmd(cmd): """Execs shell cmd and raises an exception if the cmd fails""" if (os.system(cmd)): - raise Exception, "cmd failed "+cmd + raise Exception("cmd failed "+cmd) def copy_file(src_file_abs_path, dst_file_abs_path): @@ -199,19 +199,19 @@ def copy_file(src_file_abs_path, dst_file_abs_path): # Copy the file over shutil.copy2(src_file_abs_path, dst_file_abs_path) -def bzip_file(filename): - """ Bzip's the file in place. The original file is replaced with a bzip'd version of itself +def xz_file(filename): + """ XZ compresses the file in place. The original file is replaced with the xz compressed version of itself assumes the path is absolute""" - exec_shell_cmd('bzip2 -z9 "' + filename+'"') - os.rename(filename+".bz2",filename) + exec_shell_cmd('xz --compress --x86 --lzma2 --format=xz --check=crc64 "' + filename+'"') + os.rename(filename+".xz",filename) -def bunzip_file(filename): - """ Bzip's the file in palce. The original file is replaced with a bunzip'd version of itself. - doesn't matter if the filename ends in .bz2 or not""" - if not filename.endswith(".bz2"): - os.rename(filename, filename+".bz2") - filename=filename+".bz2" - exec_shell_cmd('bzip2 -d "' + filename+'"') +def xzunzip_file(filename): + """ xz decompresses the file in palce. The original file is replaced with a xz decompressed version of itself. + doesn't matter if the filename ends in .xz or not""" + if not filename.endswith(".xz"): + os.rename(filename, filename+".xz") + filename=filename+".xz" + exec_shell_cmd('xz -d "' + filename+'"') def extract_mar(filename, work_dir): @@ -241,14 +241,14 @@ def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, pa if not os.path.exists(patch_file_dir): os.makedirs(patch_file_dir) - # Create bzip'd patch file + # Create xz'd patch file exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path) - bzip_file(patch_file_abs_path) + xz_file(patch_file_abs_path) - # Create bzip's full file + # Create xz full file full_file_abs_path = os.path.join(patch_info.work_dir, to_marfile_entry.name) shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path) - bzip_file(full_file_abs_path) + xz_file(full_file_abs_path) if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path): # Patch is smaller than file. Remove the file and add patch to manifest @@ -304,13 +304,17 @@ def process_explicit_remove_files(dir_path, patch_info): list_file_path = os.path.join(dir_path, "Contents/Resources/removed-files") if (os.path.exists(list_file_path)): - list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist + fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd()) + os.close(fd) + exec_shell_cmd('xz -k -d --stdout "' + list_file_path + '" > "'+tmppath+'"') + list_file = open(tmppath) lines = [] for line in list_file: lines.append(line.strip()) - list_file.close() + list_file.close() + os.remove(tmppath) lines.sort(reverse=True) for line in lines: # Exclude any blank and comment lines. @@ -406,7 +410,7 @@ def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch patch_info.create_manifest_files() # And construct the mar - mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ') + mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+' '.join(patch_info.archive_files) exec_shell_cmd(mar_cmd) # Copy mar to final destination @@ -431,11 +435,18 @@ def get_buildid(work_dir): print 'WARNING: application.ini not found, cannot find build ID' return '' - file = bz2.BZ2File(ini) + fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd()) + os.close(fd) + exec_shell_cmd('xz -k -d --stdout "' + ini + '" > "'+tmppath+'"') + file = open(tmppath) for line in file: if line.find('BuildID') == 0: + file.close() + os.remove(tmppath) return line.strip().split('=')[1] - print 'WARNING: cannot find build ID in application.ini' + print('WARNING: cannot find build ID in application.ini') + file.close() + os.remove(tmppath) return '' def decode_filename(filepath): @@ -486,7 +497,7 @@ def create_partial_patches(patches): extract_mar(from_filename,work_dir_from) from_decoded = decode_filename(from_filename) from_buildid = get_buildid(work_dir_from) - from_shasum = sha.sha(open(from_filename).read()).hexdigest() + from_shasum = hashlib.sha1(open(from_filename, "rb").read()).hexdigest() from_size = str(os.path.getsize(to_filename)) # Extract to mar into to dir @@ -495,14 +506,14 @@ def create_partial_patches(patches): extract_mar(to_filename, work_dir_to) to_decoded = decode_filename(from_filename) to_buildid = get_buildid(work_dir_to) - to_shasum = sha.sha(open(to_filename).read()).hexdigest() + to_shasum = hashlib.sha1(open(to_filename, 'rb').read()).hexdigest() to_size = str(os.path.getsize(to_filename)) mar_extract_time = time.time() partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['update.manifest','updatev2.manifest','updatev3.manifest'],[]),forced_updates,['channel-prefs.js','update-settings.ini']) partial_buildid = to_buildid - partial_shasum = sha.sha(open(partial_filename).read()).hexdigest() + partial_shasum = hashlib.sha1(open(partial_filename, "rb").read()).hexdigest() partial_size = str(os.path.getsize(partial_filename)) metadata.append({ diff --git a/tools/update-packaging/test/catmanifest.sh b/tools/update-packaging/test/catmanifest.sh index 354c3ff411..e6349d47a3 100755 --- a/tools/update-packaging/test/catmanifest.sh +++ b/tools/update-packaging/test/catmanifest.sh @@ -9,6 +9,6 @@ mkdir -p "$workdir" cp "$1" "$workdir" cd "$workdir" mar -x "$1" -mv updatev2.manifest updatev2.manifest.bz2 -bzip2 -d updatev2.manifest.bz2 +mv updatev2.manifest updatev2.manifest.xz +xz -d updatev2.manifest.xz cat updatev2.manifest diff --git a/tools/update-packaging/test/common.sh b/tools/update-packaging/test/common.sh index 7c133724e5..a9c904f037 100755 --- a/tools/update-packaging/test/common.sh +++ b/tools/update-packaging/test/common.sh @@ -10,9 +10,9 @@ # In here to use the local common.sh to allow the full mars to have unfiltered files # ----------------------------------------------------------------------------- -# By default just assume that these tools exist on our path +# By default just assume that these tools exist in our path MAR=${MAR:-mar} -BZIP2=${BZIP2:-bzip2} +XZ=${XZ:-xz} MBSDIFF=${MBSDIFF:-mbsdiff} # ----------------------------------------------------------------------------- diff --git a/tools/update-packaging/test/diffmar.sh b/tools/update-packaging/test/diffmar.sh index 771b6b0ce1..f1dfc7fbdd 100755 --- a/tools/update-packaging/test/diffmar.sh +++ b/tools/update-packaging/test/diffmar.sh @@ -30,19 +30,19 @@ cp "$2" "$todir" cd "$fromdir" mar -x "$1" rm "$1" -mv updatev2.manifest updatev2.manifest.bz2 -bzip2 -d updatev2.manifest.bz2 -mv updatev3.manifest updatev3.manifest.bz2 -bzip2 -d updatev3.manifest.bz2 +mv updatev2.manifest updatev2.manifest.xz +xz -d updatev2.manifest.xz +mv updatev3.manifest updatev3.manifest.xz +xz -d updatev3.manifest.xz ls $lsargs > files.txt cd "$todir" mar -x "$2" rm "$2" -mv updatev2.manifest updatev2.manifest.bz2 -bzip2 -d updatev2.manifest.bz2 -mv updatev3.manifest updatev3.manifest.bz2 -bzip2 -d updatev3.manifest.bz2 +mv updatev2.manifest updatev2.manifest.xz +xz -d updatev2.manifest.xz +mv updatev3.manifest updatev3.manifest.xz +xz -d updatev3.manifest.xz ls $lsargs > files.txt echo "diffing $fromdir and $todir" diff --git a/tools/update-packaging/test/make_full_update.sh b/tools/update-packaging/test/make_full_update.sh index f2907da9ee..d6a01dcb76 100755 --- a/tools/update-packaging/test/make_full_update.sh +++ b/tools/update-packaging/test/make_full_update.sh @@ -94,7 +94,7 @@ for ((i=0; $i<$num_files; i=$i+1)); do dir=$(dirname "$f") mkdir -p "$workdir/$dir" - $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" + $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f" copy_perm "$targetdir/$f" "$workdir/$f" targetfiles="$targetfiles \"$f\"" @@ -105,8 +105,8 @@ notice "" notice "Adding file and directory remove instructions from file 'removed-files'" append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3" -$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" -$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" +$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2" +$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" eval "$MAR -C \"$workdir\" -c output.mar $targetfiles" mv -f "$workdir/output.mar" "$archive" diff --git a/tools/update-packaging/unwrap_full_update.pl b/tools/update-packaging/unwrap_full_update.pl index ead1157db9..90aa8ee726 100755 --- a/tools/update-packaging/unwrap_full_update.pl +++ b/tools/update-packaging/unwrap_full_update.pl @@ -13,7 +13,7 @@ use Getopt::Std; -my ($MAR, $BZIP2, $archive, @marentries, @marfiles); +my ($MAR, $XZ, $BZIP2, $MAR_OLD_FORMAT, $archive, @marentries, @marfiles); if (defined($ENV{"MAR"})) { $MAR = $ENV{"MAR"}; @@ -29,6 +29,18 @@ else { $BZIP2 = "bzip2"; } +if (defined($ENV{"XZ"})) { + $XZ = $ENV{"XZ"}; +} +else { + if (system("xz --version > /dev/null 2>&1") != 0) { + die("The xz executable must be in the path."); + } + else { + $XZ = "xz"; + } +} + sub print_usage { print "Usage: unwrap_full_update.pl [OPTIONS] ARCHIVE\n\n"; @@ -50,18 +62,43 @@ $archive = $ARGV[0]; $? && die("Couldn't run \"$MAR\" -t"); -shift @marentries; +if (system("$MAR -x \"$archive\"") != 0) { + die "Couldn't run $MAR -x"; +} + +# Try to determine if the mar file contains bzip2 compressed files and if not +# assume that the mar file contains lzma compressed files. The updatev3.manifest +# file is checked since a valid mar file must have this file in the root path. +open(my $testfilename, "updatev3.manifest") or die $!; +binmode($testfilename); +read($testfilename, my $bytes, 3); +if ($bytes eq "BZh") { + $MAR_OLD_FORMAT = 1; +} else { + undef $MAR_OLD_FORMAT; +} +close $testfilename; -system("$MAR -x \"$archive\"") == 0 || die "Couldn't run $MAR -x"; +shift @marentries; foreach (@marentries) { tr/\n\r//d; my @splits = split(/\t/,$_); my $file = $splits[2]; - system("mv \"$file\" \"$file.bz2\"") == 0 || - die "Couldn't mv \"$file\""; - system("\"$BZIP2\" -d \"$file.bz2\"") == 0 || - die "Couldn't decompress \"$file\""; + print "Decompressing: " . $file . "\n"; + if ($MAR_OLD_FORMAT == 1) { + system("mv \"$file\" \"$file.bz2\"") == 0 || + die "Couldn't mv \"$file\""; + system("\"$BZIP2\" -d \"$file.bz2\"") == 0 || + die "Couldn't decompress \"$file\""; + } + else { + system("mv \"$file\" \"$file.xz\"") == 0 || + die "Couldn't mv \"$file\""; + system("\"$XZ\" -d \"$file.xz\"") == 0 || + die "Couldn't decompress \"$file\""; + } } +print "Finished\n"; |