From bd292aa668c4904fb67e36c9042cd0c7ce49538d Mon Sep 17 00:00:00 2001 From: klzgrad Date: Mon, 14 Jan 2019 00:42:27 -0500 Subject: [PATCH] Add source import tool --- CHROMIUM_VERSION | 1 + tools/build_test.sh | 101 ++++++++++++++++++++++++++++++ tools/build_test_stats.sh | 5 ++ tools/exclude.txt | 29 +++++++++ tools/import-upstream.sh | 28 +++++++++ tools/include.txt | 75 ++++++++++++++++++++++ tools/list-openwrt.sh | 60 ++++++++++++++++++ tools/list-tls-sni-stream.sh | 10 +++ tools/parse-pcap-stream.py | 118 +++++++++++++++++++++++++++++++++++ tools/sample-traffic.sh | 22 +++++++ 10 files changed, 449 insertions(+) create mode 100644 CHROMIUM_VERSION create mode 100755 tools/build_test.sh create mode 100755 tools/build_test_stats.sh create mode 100644 tools/exclude.txt create mode 100755 tools/import-upstream.sh create mode 100644 tools/include.txt create mode 100644 tools/list-openwrt.sh create mode 100755 tools/list-tls-sni-stream.sh create mode 100755 tools/parse-pcap-stream.py create mode 100755 tools/sample-traffic.sh diff --git a/CHROMIUM_VERSION b/CHROMIUM_VERSION new file mode 100644 index 0000000000..2234a95d8f --- /dev/null +++ b/CHROMIUM_VERSION @@ -0,0 +1 @@ +131.0.6778.86 diff --git a/tools/build_test.sh b/tools/build_test.sh new file mode 100755 index 0000000000..24be765cae --- /dev/null +++ b/tools/build_test.sh @@ -0,0 +1,101 @@ +#!/bin/sh +set -ex +cd src + +unset EXTRA_FLAGS +unset OPENWRT_FLAGS +ccache -C + +./get-clang.sh + +for i in x64 x86 arm64 arm mipsel mips64el; do + unset EXTRA_FLAGS + unset OPENWRT_FLAGS + export EXTRA_FLAGS="target_cpu=\"$i\"" + ./get-clang.sh +done + +for i in x64 x86 arm64 arm; do + unset EXTRA_FLAGS + unset OPENWRT_FLAGS + export EXTRA_FLAGS="target_cpu=\"$i\" target_os=\"android\"" + ./get-clang.sh +done + +config_openwrt() { + arch="$1" + openwrt="$2" + target_cpu="$3" + extra="$4" + export EXTRA_FLAGS="target_cpu=\"$target_cpu\" target_os=\"openwrt\" use_allocator=\"none\" use_allocator_shim=false $extra" + export OPENWRT_FLAGS="arch=$arch release=19.07.7 gcc_ver=7.5.0 $openwrt" + ./get-clang.sh +} + +config_openwrt x86_64 'target=x86 subtarget=64' x64 +config_openwrt x86 'target=x86 subtarget=generic' x86 +config_openwrt aarch64_cortex-a53 'target=sunxi subtarget=cortexa53' arm64 'arm_version=0 arm_cpu="cortex-a53"' +config_openwrt aarch64_cortex-a72 'target=mvebu subtarget=cortexa72' arm64 'arm_version=0 arm_cpu="cortex-a72"' +config_openwrt aarch64_generic 'target=armvirt subtarget=64' arm64 +config_openwrt arm_cortex-a5_vfpv4 'target=at91 subtarget=sama5' arm 'arm_version=0 arm_cpu="cortex-a5" arm_fpu="vfpv4" arm_float_abi="hard" arm_use_neon=false' +config_openwrt arm_cortex-a7_neon-vfpv4 'target=sunxi subtarget=cortexa7' arm 'arm_version=0 arm_cpu="cortex-a7" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true' +config_openwrt arm_cortex-a8_neon 'target=samsung subtarget=s5pv210' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true' +config_openwrt arm_cortex-a8_vfpv3 'target=sunxi subtarget=cortexa8' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="vfpv3" arm_float_abi="hard" arm_use_neon=false' +config_openwrt arm_cortex-a9 'target=bcm53xx' arm 'arm_version=0 arm_cpu="cortex-a9" arm_float_abi="soft" arm_use_neon=false' +config_openwrt arm_cortex-a9_neon 'target=imx6' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true' +config_openwrt arm_cortex-a9_vfpv3-d16 'target=tegra' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="vfpv3-d16" arm_float_abi="hard" arm_use_neon=false' +config_openwrt arm_cortex-a15_neon-vfpv4 'target=armvirt subtarget=32' arm 'arm_version=0 arm_cpu="cortex-a15" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true' +config_openwrt mipsel_24kc 'target=ramips subtarget=rt305x' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="24kc" use_lld=false use_gold=false' +config_openwrt mipsel_74kc 'target=ramips subtarget=rt3883' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="74kc" use_lld=false use_gold=false' +config_openwrt mipsel_mips32 'target=rb532' mipsel 'mips_arch_variant="r1" mips_float_abi="soft" use_lld=false use_gold=false' + +rm -f /tmp/trace +inotifywait -m -r -o/tmp/trace --format '%w%f %e' . & +pid=$! + +unset EXTRA_FLAGS +unset OPENWRT_FLAGS +./build.sh + +for i in x64 x86 arm64 arm mipsel mips64el; do + unset EXTRA_FLAGS + unset OPENWRT_FLAGS + export EXTRA_FLAGS="target_cpu=\"$i\"" + ./build.sh +done + +for i in x64 x86 arm64 arm; do + unset EXTRA_FLAGS + unset OPENWRT_FLAGS + export EXTRA_FLAGS="target_cpu=\"$i\" target_os=\"android\"" + ./build.sh +done + +build_openwrt() { + arch="$1" + openwrt="$2" + target_cpu="$3" + extra="$4" + export EXTRA_FLAGS="target_cpu=\"$target_cpu\" target_os=\"openwrt\" use_allocator=\"none\" use_allocator_shim=false $extra" + export OPENWRT_FLAGS="arch=$arch release=19.07.7 gcc_ver=7.5.0 $openwrt" + ./build.sh +} + +build_openwrt x86_64 'target=x86 subtarget=64' x64 +build_openwrt x86 'target=x86 subtarget=generic' x86 +build_openwrt aarch64_cortex-a53 'target=sunxi subtarget=cortexa53' arm64 'arm_version=0 arm_cpu="cortex-a53"' +build_openwrt aarch64_cortex-a72 'target=mvebu subtarget=cortexa72' arm64 'arm_version=0 arm_cpu="cortex-a72"' +build_openwrt aarch64_generic 'target=armvirt subtarget=64' arm64 +build_openwrt arm_cortex-a5_vfpv4 'target=at91 subtarget=sama5' arm 'arm_version=0 arm_cpu="cortex-a5" arm_fpu="vfpv4" arm_float_abi="hard" arm_use_neon=false' +build_openwrt arm_cortex-a7_neon-vfpv4 'target=sunxi subtarget=cortexa7' arm 'arm_version=0 arm_cpu="cortex-a7" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true' +build_openwrt arm_cortex-a8_neon 'target=samsung subtarget=s5pv210' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true' +build_openwrt arm_cortex-a8_vfpv3 'target=sunxi subtarget=cortexa8' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="vfpv3" arm_float_abi="hard" arm_use_neon=false' +build_openwrt arm_cortex-a9 'target=bcm53xx' arm 'arm_version=0 arm_cpu="cortex-a9" arm_float_abi="soft" arm_use_neon=false' +build_openwrt arm_cortex-a9_neon 'target=imx6' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true' +build_openwrt arm_cortex-a9_vfpv3-d16 'target=tegra' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="vfpv3-d16" arm_float_abi="hard" arm_use_neon=false' +build_openwrt arm_cortex-a15_neon-vfpv4 'target=armvirt subtarget=32' arm 'arm_version=0 arm_cpu="cortex-a15" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true' +build_openwrt mipsel_24kc 'target=ramips subtarget=rt305x' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="24kc" use_lld=false use_gold=false' +build_openwrt mipsel_74kc 'target=ramips subtarget=rt3883' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="74kc" use_lld=false use_gold=false' +build_openwrt mipsel_mips32 'target=rb532' mipsel 'mips_arch_variant="r1" mips_float_abi="soft" use_lld=false use_gold=false' + +kill $pid diff --git a/tools/build_test_stats.sh b/tools/build_test_stats.sh new file mode 100755 index 0000000000..78199db48a --- /dev/null +++ b/tools/build_test_stats.sh @@ -0,0 +1,5 @@ +#!/bin/sh +for i in /tmp/trace.*; do + cut -d' ' -f1 $i | LC_ALL=C sort -u | sed 's/\/$//' | LC_ALL=C sort -u >$i.sorted +done +cat /tmp/trace.*.sorted | LC_ALL=C sort -u >/tmp/detected-files diff --git a/tools/exclude.txt b/tools/exclude.txt new file mode 100644 index 0000000000..9307606b1e --- /dev/null +++ b/tools/exclude.txt @@ -0,0 +1,29 @@ +.gitignore +*_unittest.cc +*_unittest.mm +*_unittest.nc +*_perftest.cc +*[!s]_test.cc +*[a-hj-z]s_test.cc +*fuzz* +*org/chromium* +*.golden +*.javap* +*.pyc +build/linux/debian_* +base/tracing/test +net/data/[!s]* +net/data/s[!s]* +net/data/ssl/[!ce]* +net/data/ssl/c[!h]* +net/http/transport_security_state_static.json +net/third_party/nist-pkits +third_party/boringssl/src/crypto/*.txt +third_party/boringssl/src/crypto/hpke/test-vectors.json +third_party/boringssl/src/crypto/cipher_extra/test +third_party/boringssl/src/pki/testdata +third_party/boringssl/src/ssl/test +third_party/boringssl/src/util +third_party/boringssl/src/third_party/googletest +third_party/boringssl/src/third_party/wycheproof_testvectors +third_party/libc++/src/test diff --git a/tools/import-upstream.sh b/tools/import-upstream.sh new file mode 100755 index 0000000000..c3737e7622 --- /dev/null +++ b/tools/import-upstream.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -ex +have_version=$(cut -d= -f2 src/chrome/VERSION | tr '\n' . | cut -d. -f1-4) +want_version=$(cat CHROMIUM_VERSION) +if [ "$have_version" = "$want_version" ]; then + exit 0 +fi +name="chromium-$want_version" +tarball="$name.tar.xz" +url="https://commondatastorage.googleapis.com/chromium-browser-official/$tarball" +root=$(git rev-list --max-parents=0 HEAD) +branch=$(git branch --show-current) +git config core.autocrlf false +git config core.safecrlf false +git -c advice.detachedHead=false checkout $root +rm -rf src +git checkout "$branch" -- tools +sed -i "s/^\^/$name\//" tools/include.txt +if [ -f "/tmp/$tarball" ]; then + cat "/tmp/$tarball" | tar xJf - --wildcards --wildcards-match-slash -T tools/include.txt -X tools/exclude.txt +else + curl "$url" -o- | tar xJf - --wildcards --wildcards-match-slash -T tools/include.txt -X tools/exclude.txt +fi +mv "$name" src +git rm --quiet --force -r tools +git add src +git commit --quiet --amend -m "Import $name" --date=now +git rebase --onto HEAD "$root" "$branch" diff --git a/tools/include.txt b/tools/include.txt new file mode 100644 index 0000000000..dbf392db31 --- /dev/null +++ b/tools/include.txt @@ -0,0 +1,75 @@ +^.clang-format +^.gitattributes +^.gitignore +^.gn +^AUTHORS +^BUILD.gn +^DEPS +^LICENSE +^base +^build +^build_overrides/build.gni +^build_overrides/partition_alloc.gni +^buildtools/deps_revisions.gni +^buildtools/third_party/eu-strip/bin/eu-strip +^buildtools/third_party/libc++/BUILD.gn +^buildtools/third_party/libc++/__assertion_handler +^buildtools/third_party/libc++/__config_site +^buildtools/third_party/libc++abi/BUILD.gn +^buildtools/third_party/libc++abi/cxa_demangle_stub.cc +^buildtools/third_party/libunwind/BUILD.gn +^chrome/VERSION +^chrome/android/profiles/newest.txt +^chrome/app/theme/chromium/BRANDING +^chrome/build/*.txt +^chrome/version.gni +^components/miracle_parameter +^components/nacl/toolchain.gni +^components/network_time +^components/version_info +^crypto +^ios/features.gni +^ipc/ipc_param_traits.h +^net +^testing/gtest/include/gtest/gtest_prod.h +^third_party/abseil-cpp +^third_party/angle/dotfile_settings.gni +^third_party/angle/src/commit_id.py +^third_party/angle/scripts/file_exists.py +^third_party/apple_apsl +^third_party/ashmem +^third_party/boringssl +^third_party/brotli +^third_party/closure_compiler/closure_args.gni +^third_party/closure_compiler/compile_js.gni +^third_party/cpu_features +^third_party/depot_tools/cpplint.py +^third_party/depot_tools/download_from_google_storage.py +^third_party/depot_tools/subprocess2.py +^third_party/googletest/BUILD.gn +^third_party/googletest/src/googletest/include/gtest/gtest_prod.h +^third_party/icu/config.gni +^third_party/jni_zero +^third_party/libc++ +^third_party/libc++abi +^third_party/libevent +^third_party/libunwind +^third_party/lss/linux_syscall_support.h +^third_party/modp_b64 +^third_party/nasm +^third_party/perfetto/include/perfetto/tracing/traced_value_forward.h +^third_party/protobuf/BUILD.gn +^third_party/protobuf/proto_library.gni +^third_party/protobuf/proto_sources.gni +^third_party/protobuf/src +^third_party/zlib +^third_party/zstd +^tools/cfi +^tools/clang/scripts/update.py +^tools/diagnosis +^tools/grit +^tools/gritsettings +^tools/protoc_wrapper +^tools/update_pgo_profiles.py +^tools/win/DebugVisualizers +^url diff --git a/tools/list-openwrt.sh b/tools/list-openwrt.sh new file mode 100644 index 0000000000..c93a3ab7da --- /dev/null +++ b/tools/list-openwrt.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# $version can be 21.02 or 19.07. +version=19.07.7 +if [ ! -d /tmp/openwrt ]; then + cd /tmp + git clone https://github.com/openwrt/openwrt.git + cd openwrt +fi +cd /tmp/openwrt +git -c advice.detachedHead=false checkout v$version +export TOPDIR=$PWD +cd target/linux +>targets.git +for target in *; do + [ -d $target ] || continue + subtargets=$(make -C $target --no-print-directory DUMP=1 TARGET_BUILD=1 val.SUBTARGETS 2>/dev/null) + [ "$subtargets" ] || subtargets=generic + for subtarget in $subtargets; do + echo $(make -C $target --no-print-directory DUMP=1 TARGET_BUILD=1 SUBTARGET=$subtarget 2>/dev/null | egrep '^(Target:|Target-Arch-Packages:)' | cut -d: -f2) >>targets.git + done +done + +targets=$(curl -s https://downloads.openwrt.org/releases/$version/targets/ | grep 'targets.sdk +for target in $targets; do + subtargets=$(curl -s https://downloads.openwrt.org/releases/$version/targets/$target/ | grep '>targets.sdk + done +done + +cat >parse-targets.py <