Port the libvpx mozbuild generator to aom

This is a port of the libvpx scripts, themselves a port of Chromium's scripts to generate an external build description using hooks in the upstream configure and make scripts. The libaom library is a fork of libvpx so we can use a similar approach.

The upstream source is located in $(topsrc_dir)/third_party/aom but the build description and any patches are under the media directory with the other codecs, similar to how zlib works. Config files and headers generated by the upstream build system are also under $(topsrc_dir)/media/libaom/.

4 files changed, 516 insertions, 0 deletions

diff --git a/media/libaom/README_MCP b/media/libaom/README_MCP
new file mode 100644
index 0000000000..b3046e6319
--- /dev/null
+++ b/media/libaom/README_MCP
@@ -0,0 +1,11 @@
+This directory contains build files for the aom video
+codec reference implementation. The actual library
+source is in $TOPSRCDIR/third_party/aom/
+
+Any patches or additional configuration to be applied to the
+upstream source should be kept here in the media/libaom
+directory.
+
+The upstream aom git repository is:
+
+    https://aomedia.googlesource.com/aom
diff --git a/media/libaom/generate_sources_mozbuild.sh b/media/libaom/generate_sources_mozbuild.sh
new file mode 100755
index 0000000000..a8c0ba83a3
--- /dev/null
+++ b/media/libaom/generate_sources_mozbuild.sh
@@ -0,0 +1,270 @@
+#!/bin/bash -e
+#
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Modified from chromium/src/third_party/libaom/generate_gni.sh
+
+# This script is used to generate sources.mozbuild and files in the
+# config/platform directories needed to build libaom.
+# Every time libaom source code is updated just run this script.
+#
+# Usage:
+# $ ./generate_sources_mozbuild.sh
+
+export LC_ALL=C
+BASE_DIR=$(pwd)
+LIBAOM_SRC_DIR="../../third_party/aom"
+LIBAOM_CONFIG_DIR="config"
+
+# Print license header.
+# $1 - Output base name
+function write_license {
+  echo "# This file is automatically generated. Do not edit." >> $1
+  echo "" >> $1
+}
+
+# Search for source files with the same basename.
+# The build does not support duplicate file names.
+function find_duplicates {
+  local readonly duplicate_file_names=$(find \
+    $BASE_DIR/$LIBAOM_SRC_DIR/vp8 \
+    $BASE_DIR/$LIBAOM_SRC_DIR/vp9 \
+    $BASE_DIR/$LIBAOM_SRC_DIR/av1 \
+    $BASE_DIR/$LIBAOM_SRC_DIR/aom \
+    $BASE_DIR/$LIBAOM_SRC_DIR/aom_dsp \
+    -type f -name \*.c  | xargs -I {} basename {} | sort | uniq -d \
+  )
+
+  if [ -n "${duplicate_file_names}" ]; then
+    echo "ERROR: DUPLICATE FILES FOUND"
+    for file in  ${duplicate_file_names}; do
+      find \
+        $BASE_DIR/$LIBAOM_SRC_DIR/vp8 \
+        $BASE_DIR/$LIBAOM_SRC_DIR/vp9 \
+        $BASE_DIR/$LIBAOM_SRC_DIR/av1 \
+        $BASE_DIR/$LIBAOM_SRC_DIR/aom \
+        $BASE_DIR/$LIBAOM_SRC_DIR/aom_dsp \
+        -name $file
+    done
+    exit 1
+  fi
+}
+
+# Generate sources.mozbuild with a list of source files.
+# $1 - Array name for file list. This is processed with 'declare' below to
+#      regenerate the array locally.
+# $2 - Variable name.
+# $3 - Output file.
+function write_sources {
+  # Convert the first argument back in to an array.
+  declare -a file_list=("${!1}")
+
+  echo "  '$2': [" >> "$3"
+  for f in $file_list
+  do
+    echo "    '$LIBAOM_SRC_DIR/$f'," >> "$3"
+  done
+  echo "]," >> "$3"
+}
+
+# Convert a list of source files into sources.mozbuild.
+# $1 - Input file.
+# $2 - Output prefix.
+function convert_srcs_to_project_files {
+  # Do the following here:
+  # 1. Filter .c, .h, .s, .S and .asm files.
+  # 3. Convert .asm.s to .asm because moz.build will do the conversion.
+
+  local source_list=$(grep -E '(\.c|\.h|\.S|\.s|\.asm)$' $1)
+
+  # Remove aom_config.c.
+  source_list=$(echo "$source_list" | grep -v 'aom_config\.c')
+
+  # The actual ARM files end in .asm. We have rules to translate them to .S
+  source_list=$(echo "$source_list" | sed s/\.asm\.s$/.asm/)
+
+  # Exports - everything in aom, aom_mem, aom_ports, aom_scale
+  local exports_list=$(echo "$source_list" | \
+    egrep '^(aom|aom_mem|aom_ports|aom_scale)/.*h$')
+  # but not anything in one level down, like 'internal'
+  exports_list=$(echo "$exports_list" | egrep -v '/(internal|src)/')
+  # or any of the other internal-ish header files.
+  exports_list=$(echo "$exports_list" | egrep -v '/(emmintrin_compat.h|mem_.*|msvc.h|aom_once.h)$')
+
+  # Remove these files from the main list.
+  source_list=$(comm -23 <(echo "$source_list") <(echo "$exports_list"))
+
+  # Write a single file that includes all source files for all archs.
+  local c_sources=$(echo "$source_list" | egrep '.(asm|c)$')
+  local exports_sources=$(echo "$exports_list" | egrep '.h$')
+
+  write_sources exports_sources ${2}_EXPORTS "$BASE_DIR/sources.mozbuild"
+  write_sources c_sources ${2}_SOURCES "$BASE_DIR/sources.mozbuild"
+}
+
+# Clean files from previous make.
+function make_clean {
+  make clean > /dev/null
+  rm -f libaom_srcs.txt
+}
+
+# Print the configuration.
+# $1 - Header file directory.
+function print_config {
+  $BASE_DIR/lint_config.sh -p \
+    -h $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_config.h \
+    -a $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_config.asm
+}
+
+# Generate *_rtcd.h files.
+# $1 - Header file directory.
+# $2 - Architecture.
+# $3 - Optional - any additional arguments to pass through.
+function gen_rtcd_header {
+  echo "Generate $LIBAOM_CONFIG_DIR/$1/*_rtcd.h files."
+
+  rm -rf $TEMP_DIR/libaom.config
+  $BASE_DIR/lint_config.sh -p \
+    -h $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_config.h \
+    -a $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_config.asm \
+    -o $TEMP_DIR/libaom.config
+
+  $BASE_DIR/$LIBAOM_SRC_DIR/build/make/rtcd.pl \
+    --arch=$2 \
+    --sym=aom_rtcd $DISABLE_AVX $3 \
+    --config=$TEMP_DIR/libaom.config \
+    $BASE_DIR/$LIBAOM_SRC_DIR/av1/common/av1_rtcd_defs.pl \
+    > $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/av1_rtcd.h
+
+  $BASE_DIR/$LIBAOM_SRC_DIR/build/make/rtcd.pl \
+    --arch=$2 \
+    --sym=aom_scale_rtcd $DISABLE_AVX $3 \
+    --config=$TEMP_DIR/libaom.config \
+    $BASE_DIR/$LIBAOM_SRC_DIR/aom_scale/aom_scale_rtcd.pl \
+    > $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_scale_rtcd.h
+
+  $BASE_DIR/$LIBAOM_SRC_DIR/build/make/rtcd.pl \
+    --arch=$2 \
+    --sym=aom_dsp_rtcd $DISABLE_AVX $3 \
+    --config=$TEMP_DIR/libaom.config \
+    $BASE_DIR/$LIBAOM_SRC_DIR/aom_dsp/aom_dsp_rtcd_defs.pl \
+    > $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_dsp_rtcd.h
+
+  rm -rf $TEMP_DIR/libaom.config
+}
+
+# Generate Config files. "--enable-external-build" must be set to skip
+# detection of capabilities on specific targets.
+# $1 - Header file directory.
+# $2 - Config command line.
+function gen_config_files {
+  ./configure $2 #> /dev/null
+
+  # Disable HAVE_UNISTD_H.
+  ( echo '/HAVE_UNISTD_H'; echo 'd' ; echo 'w' ; echo 'q' ) | ed -s aom_config.h
+
+  local ASM_CONV=ads2gas.pl
+
+  # Generate aom_config.asm.
+  if [[ "$1" == *x64* ]] || [[ "$1" == *ia32* ]]; then
+    egrep "#define [A-Z0-9_]+ [01]" aom_config.h | awk '{print "%define " $2 " " $3}' > aom_config.asm
+  else
+    egrep "#define [A-Z0-9_]+ [01]" aom_config.h | awk '{print $2 " EQU " $3}' | perl $BASE_DIR/$LIBAOM_SRC_DIR/build/make/$ASM_CONV > aom_config.asm
+  fi
+
+  mkdir -p $BASE_DIR/$LIBAOM_CONFIG_DIR/$1
+  cp aom_config.* $BASE_DIR/$LIBAOM_CONFIG_DIR/$1
+  make_clean
+  rm -rf aom_config.*
+}
+
+find_duplicates
+
+echo "Creating temporary directory."
+TEMP_DIR="$BASE_DIR/.temp"
+rm -rf $TEMP_DIR
+cp -R $LIBAOM_SRC_DIR $TEMP_DIR
+cd $TEMP_DIR
+
+echo "Generating config files."
+all_platforms="--enable-external-build --disable-examples --disable-docs --disable-unit-tests"
+all_platforms="${all_platforms} --size-limit=8192x4608 --enable-pic"
+x86_platforms="--enable-postproc --as=yasm"
+arm_platforms="--enable-runtime-cpu-detect --enable-realtime-only"
+gen_config_files linux/x64 "--target=x86_64-linux-gcc ${all_platforms} ${x86_platforms}"
+gen_config_files linux/ia32 "--target=x86-linux-gcc ${all_platforms} ${x86_platforms}"
+gen_config_files mac/x64 "--target=x86_64-darwin9-gcc ${all_platforms} ${x86_platforms}"
+gen_config_files win/x64 "--target=x86_64-win64-vs12 ${all_platforms} ${x86_platforms}"
+gen_config_files win/ia32 "--target=x86-win32-gcc ${all_platforms} ${x86_platforms}"
+
+gen_config_files linux/arm "--target=armv7-linux-gcc ${all_platforms} ${arm_platforms}"
+
+gen_config_files generic "--target=generic-gnu ${all_platforms}"
+
+echo "Remove temporary directory."
+cd $BASE_DIR
+rm -rf $TEMP_DIR
+
+echo "Create temporary directory."
+TEMP_DIR="$BASE_DIR/.temp"
+rm -rf $TEMP_DIR
+cp -R $LIBAOM_SRC_DIR $TEMP_DIR
+cd $TEMP_DIR
+
+gen_rtcd_header linux/x64 x86_64
+gen_rtcd_header linux/ia32 x86
+gen_rtcd_header mac/x64 x86_64
+gen_rtcd_header win/x64 x86_64
+gen_rtcd_header win/ia32 x86
+
+gen_rtcd_header linux/arm armv7
+
+gen_rtcd_header generic generic
+
+echo "Prepare Makefile."
+./configure --target=generic-gnu > /dev/null
+make_clean
+
+# Remove existing source files.
+rm -rf $BASE_DIR/sources.mozbuild
+write_license $BASE_DIR/sources.mozbuild
+echo "files = {" >> $BASE_DIR/sources.mozbuild
+
+echo "Generate X86_64 source list."
+config=$(print_config linux/x64)
+make_clean
+make libaom_srcs.txt target=libs $config > /dev/null
+convert_srcs_to_project_files libaom_srcs.txt X64
+
+# Copy aom_version.h once. The file is the same for all platforms.
+cp aom_version.h $BASE_DIR/$LIBAOM_CONFIG_DIR
+
+echo "Generate IA32 source list."
+config=$(print_config linux/ia32)
+make_clean
+make libaom_srcs.txt target=libs $config > /dev/null
+convert_srcs_to_project_files libaom_srcs.txt IA32
+
+echo "Generate ARM source list."
+config=$(print_config linux/arm)
+make_clean
+make libaom_srcs.txt target=libs $config > /dev/null
+convert_srcs_to_project_files libaom_srcs.txt ARM
+
+echo "Generate generic source list."
+config=$(print_config generic)
+make_clean
+make libaom_srcs.txt target=libs $config > /dev/null
+convert_srcs_to_project_files libaom_srcs.txt GENERIC
+
+echo "}" >> $BASE_DIR/sources.mozbuild
+
+echo "Remove temporary directory."
+cd $BASE_DIR
+rm -rf $TEMP_DIR
+
+cd $BASE_DIR/$LIBAOM_SRC_DIR
+
+cd $BASE_DIR
diff --git a/media/libaom/lint_config.sh b/media/libaom/lint_config.sh
new file mode 100755
index 0000000000..1a6c96dfbb
--- /dev/null
+++ b/media/libaom/lint_config.sh
@@ -0,0 +1,112 @@
+#!/bin/bash -e
+#
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This script is used to compare vpx_config.h and vpx_config.asm to
+# verify the two files match.
+#
+# Arguments:
+#
+# -h - C Header file.
+# -a - ASM file.
+# -p - Print the options if correct.
+# -o - Output file.
+#
+# Usage:
+#
+# # Compare the two configuration files and output the final results.
+# ./lint_config.sh -h vpx_config.h -a vpx_config.asm -o libvpx.config -p
+
+export LC_ALL=C
+print_final="no"
+
+while getopts "h:a:o:p" flag
+do
+  if [ "$flag" = "h" ]; then
+    header_file=$OPTARG
+  elif [ "$flag" = "a" ]; then
+    asm_file=$OPTARG
+  elif [ "$flag" = "o" ]; then
+    out_file=$OPTARG
+  elif [ "$flag" = "p" ]; then
+    print_final="yes"
+  fi
+done
+
+if [ -z "$header_file" ]; then
+  echo "Header file not specified."
+  false
+  exit
+fi
+
+if [ -z "$asm_file" ]; then
+  echo "ASM file not specified."
+  false
+  exit
+fi
+
+# Concat header file and assembly file and select those ended with 0 or 1.
+combined_config="$(cat $header_file $asm_file | grep -E ' +[01] *$')"
+
+# Extra filtering for known exceptions.
+combined_config="$(echo "$combined_config" | grep -v WIDE_REFERENCE)"
+combined_config="$(echo "$combined_config" | grep -v ARCHITECTURE)"
+combined_config="$(echo "$combined_config" | grep -v DO1STROUNDING)"
+
+# Remove all spaces.
+combined_config="$(echo "$combined_config" | sed 's/[ \t]//g')"
+
+# Remove #define in the header file.
+combined_config="$(echo "$combined_config" | sed 's/.*define//')"
+
+# Remove equ in the ASM file.
+combined_config="$(echo "$combined_config" | sed 's/\.equ//')" # gas style
+combined_config="$(echo "$combined_config" | sed 's/equ//')" # rvds style
+combined_config="$(echo "$combined_config" | sed 's/\.set//')" # apple style
+
+# Remove %define in YASM ASM files.
+combined_config="$(echo "$combined_config" | sed 's/%define\s *//')" # yasm style
+
+# Remove useless comma in gas style assembly file.
+combined_config="$(echo "$combined_config" | sed 's/,//')"
+
+# Substitute 0 with =no.
+combined_config="$(echo "$combined_config" | sed 's/0$/=no/')"
+
+# Substitute 1 with =yes.
+combined_config="$(echo "$combined_config" | sed 's/1$/=yes/')"
+
+# Find the mismatch variables.
+odd_config="$(echo "$combined_config" | sort | uniq -u)"
+odd_vars="$(echo "$odd_config" | sed 's/=.*//' | uniq)"
+
+for var in $odd_vars; do
+  echo "Error: Configuration mismatch for $var."
+  echo "Header file: $header_file"
+  echo "$(cat -n $header_file | grep "$var[ \t]")"
+  echo "Assembly file: $asm_file"
+  echo "$(cat -n $asm_file | grep "$var[ \t]")"
+  echo ""
+done
+
+if [ -n "$odd_vars" ]; then
+  false
+  exit
+fi
+
+if [ "$print_final" = "no" ]; then
+  exit
+fi
+
+# Do some additional filter to make libvpx happy.
+combined_config="$(echo "$combined_config" | grep -v ARCH_X86=no)"
+combined_config="$(echo "$combined_config" | grep -v ARCH_X86_64=no)"
+
+# Print out the unique configurations.
+if [ -n "$out_file" ]; then
+  echo "$combined_config" | sort | uniq > $out_file
+else
+  echo "$combined_config" | sort | uniq
+fi
diff --git a/media/libaom/moz.build b/media/libaom/moz.build
new file mode 100644
index 0000000000..2efe9eeb8a
--- /dev/null
+++ b/media/libaom/moz.build
@@ -0,0 +1,123 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files('*'):
+    BUG_COMPONENT = ('Core', 'Audio/Video')
+
+include('sources.mozbuild')
+
+# Linux, Mac and Win share file lists for x86* but not configurations.
+if CONFIG['CPU_ARCH'] == 'x86_64':
+    EXPORTS.aom += files['X64_EXPORTS']
+    SOURCES += files['X64_SOURCES']
+    if CONFIG['OS_TARGET'] == 'WINNT':
+        ASFLAGS += [ '-I%s/media/libaom/config/win/x64/' % TOPSRCDIR ]
+        CFLAGS += [ '-I%s/media/libaom/config/win/x64/' % TOPSRCDIR ]
+    elif CONFIG['OS_TARGET'] == 'Darwin':
+        ASFLAGS += [ '-I%s/media/libaom/config/mac/x64/' % TOPSRCDIR ]
+        CFLAGS += [ '-I%s/media/libaom/config/mac/x64/' % TOPSRCDIR ]
+    else: # Android, Linux, BSDs, etc.
+        ASFLAGS += [ '-I%s/media/libaom/config/linux/x64/' % TOPSRCDIR ]
+        CFLAGS += [ '-I%s/media/libaom/config/linux/x64/' % TOPSRCDIR ]
+elif CONFIG['CPU_ARCH'] == 'x86':
+    EXPORTS.aom += files['IA32_EXPORTS']
+    SOURCES += files['IA32_SOURCES']
+    if CONFIG['OS_TARGET'] == 'WINNT':
+        ASFLAGS += [ '-I%s/media/libaom/config/win/ia32/' % TOPSRCDIR ]
+        CFLAGS += [ '-I%s/media/libaom/config/win/ia32/' % TOPSRCDIR ]
+    elif CONFIG['OS_TARGET'] == 'Darwin':
+        ASFLAGS += [ '-I%s/media/libaom/config/mac/ia32/' % TOPSRCDIR ]
+        CFLAGS += [ '-I%s/media/libaom/config/mac/ia32/' % TOPSRCDIR ]
+    else: # Android, Linux, BSDs, etc.
+        ASFLAGS += [ '-I%s/media/libaom/config/linux/ia32/' % TOPSRCDIR ]
+        CFLAGS += [ '-I%s/media/libaom/config/linux/ia32/' % TOPSRCDIR ]
+elif CONFIG['CPU_ARCH'] == 'arm':
+    EXPORTS.aom += files['ARM_EXPORTS']
+    ASFLAGS += [
+        '-I%s/media/libaom/config/linux/arm/' % TOPSRCDIR,
+        '-I%s/libaom' % OBJDIR,
+    ]
+    CFLAGS += [ '-I%s/media/libaom/config/linux/arm/' % TOPSRCDIR ]
+
+    arm_asm_files = files['ARM_SOURCES']
+
+    if CONFIG['VPX_AS_CONVERSION']:
+        SOURCES += sorted([
+            "!%s.S" % f if f.endswith('.asm') else f for f in arm_asm_files
+        ])
+    else:
+        SOURCES += sorted(arm_asm_files)
+
+    for f in SOURCES:
+        if f.endswith('neon.c'):
+            SOURCES[f].flags += CONFIG['VPX_ASFLAGS']
+
+    if CONFIG['OS_TARGET'] == 'Android':
+        # For cpu-features.h
+        LOCAL_INCLUDES += [
+            '%%%s/sources/android/cpufeatures' % CONFIG['ANDROID_NDK'],
+        ]
+    if CONFIG['CLANG_CXX']:
+        ASFLAGS += [
+            '-no-integrated-as',
+        ]
+else:
+    # Generic C-only configuration
+    EXPORTS.aom += files['GENERIC_EXPORTS']
+    SOURCES += files['GENERIC_SOURCES']
+    ASFLAGS += [ '-I%s/media/libaom/config/generic/' % TOPSRCDIR ]
+    CFLAGS += [ '-I%s/media/libaom/config/generic/' % TOPSRCDIR ]
+
+# We allow warnings for third-party code that can be updated from upstream.
+ALLOW_COMPILER_WARNINGS = True
+
+FINAL_LIBRARY = 'gkmedias'
+
+if CONFIG['OS_TARGET'] == 'Android':
+    # Older versions of the Android NDK don't pre-define anything to indicate
+    # the OS they're on, so do it for them.
+    DEFINES['__linux__'] = True
+
+    if not CONFIG['MOZ_WEBRTC']:
+        SOURCES += [
+            '%%%s/sources/android/cpufeatures/cpu-features.c' % CONFIG['ANDROID_NDK'],
+        ]
+
+if CONFIG['CLANG_CL'] or not CONFIG['_MSC_VER']:
+    for f in SOURCES:
+        if f.endswith('sse2.c'):
+            SOURCES[f].flags += CONFIG['SSE2_FLAGS']
+        elif f.endswith('ssse3.c'):
+            SOURCES[f].flags += ['-mssse3']
+        elif f.endswith('sse4.c'):
+            SOURCES[f].flags += ['-msse4.1']
+        elif f.endswith('avx.c'):
+            SOURCES[f].flags += ['-mavx']
+        elif f.endswith('avx2.c'):
+            SOURCES[f].flags += ['-mavx2']
+
+# Suppress warnings in third-party code.
+if CONFIG['GNU_CC'] or CONFIG['CLANG_CL']:
+    CFLAGS += [
+        '-Wno-sign-compare',
+        '-Wno-unused-function', # so many of these warnings; just ignore them
+    ]
+if CONFIG['CLANG_CXX'] or CONFIG['CLANG_CL']:
+    CFLAGS += [
+        '-Wno-unreachable-code',
+        '-Wno-unneeded-internal-declaration',
+    ]
+
+ASFLAGS += CONFIG['VPX_ASFLAGS']
+ASFLAGS += [
+    '-I.',
+    '-I%s/third_party/aom' % TOPSRCDIR,
+]
+
+CFLAGS += [
+    '-I%s/third_party/aom' % TOPSRCDIR,
+    '-I%s/media/libaom/config' % TOPSRCDIR, # aom_version.h
+]