Add source import tool

CHROMIUM_VERSION

@@ -0,0 +1 @@
+120.0.6099.43

tools/build_test.sh

@@ -0,0 +1,101 @@
+#!/bin/sh
+set -ex
+cd src
+
+unset EXTRA_FLAGS
+unset OPENWRT_FLAGS
+ccache -C
+
+./get-clang.sh
+
+for i in x64 x86 arm64 arm mipsel mips64el; do
+  unset EXTRA_FLAGS
+  unset OPENWRT_FLAGS
+  export EXTRA_FLAGS="target_cpu=\"$i\""
+  ./get-clang.sh
+done
+
+for i in x64 x86 arm64 arm; do
+  unset EXTRA_FLAGS
+  unset OPENWRT_FLAGS
+  export EXTRA_FLAGS="target_cpu=\"$i\" target_os=\"android\""
+  ./get-clang.sh
+done
+
+config_openwrt() {
+  arch="$1"
+  openwrt="$2"
+  target_cpu="$3"
+  extra="$4"
+  export EXTRA_FLAGS="target_cpu=\"$target_cpu\" target_os=\"openwrt\" use_allocator=\"none\" use_allocator_shim=false $extra"
+  export OPENWRT_FLAGS="arch=$arch release=19.07.7 gcc_ver=7.5.0 $openwrt"
+  ./get-clang.sh
+}
+
+config_openwrt x86_64 'target=x86 subtarget=64' x64
+config_openwrt x86 'target=x86 subtarget=generic' x86
+config_openwrt aarch64_cortex-a53 'target=sunxi subtarget=cortexa53' arm64 'arm_version=0 arm_cpu="cortex-a53"'
+config_openwrt aarch64_cortex-a72 'target=mvebu subtarget=cortexa72' arm64 'arm_version=0 arm_cpu="cortex-a72"'
+config_openwrt aarch64_generic 'target=armvirt subtarget=64' arm64
+config_openwrt arm_cortex-a5_vfpv4 'target=at91 subtarget=sama5' arm 'arm_version=0 arm_cpu="cortex-a5" arm_fpu="vfpv4" arm_float_abi="hard" arm_use_neon=false'
+config_openwrt arm_cortex-a7_neon-vfpv4 'target=sunxi subtarget=cortexa7' arm 'arm_version=0 arm_cpu="cortex-a7" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true'
+config_openwrt arm_cortex-a8_neon 'target=samsung subtarget=s5pv210' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true'
+config_openwrt arm_cortex-a8_vfpv3 'target=sunxi subtarget=cortexa8' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="vfpv3" arm_float_abi="hard" arm_use_neon=false'
+config_openwrt arm_cortex-a9 'target=bcm53xx' arm 'arm_version=0 arm_cpu="cortex-a9" arm_float_abi="soft" arm_use_neon=false'
+config_openwrt arm_cortex-a9_neon 'target=imx6' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true'
+config_openwrt arm_cortex-a9_vfpv3-d16 'target=tegra' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="vfpv3-d16" arm_float_abi="hard" arm_use_neon=false'
+config_openwrt arm_cortex-a15_neon-vfpv4 'target=armvirt subtarget=32' arm 'arm_version=0 arm_cpu="cortex-a15" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true'
+config_openwrt mipsel_24kc 'target=ramips subtarget=rt305x' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="24kc" use_lld=false use_gold=false'
+config_openwrt mipsel_74kc 'target=ramips subtarget=rt3883' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="74kc" use_lld=false use_gold=false'
+config_openwrt mipsel_mips32 'target=rb532' mipsel 'mips_arch_variant="r1" mips_float_abi="soft" use_lld=false use_gold=false'
+
+rm -f /tmp/trace
+inotifywait -m -r -o/tmp/trace --format '%w%f %e' . &
+pid=$!
+
+unset EXTRA_FLAGS
+unset OPENWRT_FLAGS
+./build.sh
+
+for i in x64 x86 arm64 arm mipsel mips64el; do
+  unset EXTRA_FLAGS
+  unset OPENWRT_FLAGS
+  export EXTRA_FLAGS="target_cpu=\"$i\""
+  ./build.sh
+done
+
+for i in x64 x86 arm64 arm; do
+  unset EXTRA_FLAGS
+  unset OPENWRT_FLAGS
+  export EXTRA_FLAGS="target_cpu=\"$i\" target_os=\"android\""
+  ./build.sh
+done
+
+build_openwrt() {
+  arch="$1"
+  openwrt="$2"
+  target_cpu="$3"
+  extra="$4"
+  export EXTRA_FLAGS="target_cpu=\"$target_cpu\" target_os=\"openwrt\" use_allocator=\"none\" use_allocator_shim=false $extra"
+  export OPENWRT_FLAGS="arch=$arch release=19.07.7 gcc_ver=7.5.0 $openwrt"
+  ./build.sh
+}
+
+build_openwrt x86_64 'target=x86 subtarget=64' x64
+build_openwrt x86 'target=x86 subtarget=generic' x86
+build_openwrt aarch64_cortex-a53 'target=sunxi subtarget=cortexa53' arm64 'arm_version=0 arm_cpu="cortex-a53"'
+build_openwrt aarch64_cortex-a72 'target=mvebu subtarget=cortexa72' arm64 'arm_version=0 arm_cpu="cortex-a72"'
+build_openwrt aarch64_generic 'target=armvirt subtarget=64' arm64
+build_openwrt arm_cortex-a5_vfpv4 'target=at91 subtarget=sama5' arm 'arm_version=0 arm_cpu="cortex-a5" arm_fpu="vfpv4" arm_float_abi="hard" arm_use_neon=false'
+build_openwrt arm_cortex-a7_neon-vfpv4 'target=sunxi subtarget=cortexa7' arm 'arm_version=0 arm_cpu="cortex-a7" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true'
+build_openwrt arm_cortex-a8_neon 'target=samsung subtarget=s5pv210' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true'
+build_openwrt arm_cortex-a8_vfpv3 'target=sunxi subtarget=cortexa8' arm 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="vfpv3" arm_float_abi="hard" arm_use_neon=false'
+build_openwrt arm_cortex-a9 'target=bcm53xx' arm 'arm_version=0 arm_cpu="cortex-a9" arm_float_abi="soft" arm_use_neon=false'
+build_openwrt arm_cortex-a9_neon 'target=imx6' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true'
+build_openwrt arm_cortex-a9_vfpv3-d16 'target=tegra' arm 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="vfpv3-d16" arm_float_abi="hard" arm_use_neon=false'
+build_openwrt arm_cortex-a15_neon-vfpv4 'target=armvirt subtarget=32' arm 'arm_version=0 arm_cpu="cortex-a15" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true'
+build_openwrt mipsel_24kc 'target=ramips subtarget=rt305x' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="24kc" use_lld=false use_gold=false'
+build_openwrt mipsel_74kc 'target=ramips subtarget=rt3883' mipsel 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="74kc" use_lld=false use_gold=false'
+build_openwrt mipsel_mips32 'target=rb532' mipsel 'mips_arch_variant="r1" mips_float_abi="soft" use_lld=false use_gold=false'
+
+kill $pid

tools/build_test_stats.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+for i in /tmp/trace.*; do
+  cut -d' ' -f1 $i | LC_ALL=C sort -u | sed 's/\/$//' | LC_ALL=C sort -u >$i.sorted
+done
+cat /tmp/trace.*.sorted | LC_ALL=C sort -u >/tmp/detected-files

tools/exclude.txt

@@ -0,0 +1,22 @@
+.gitignore
+*_unittest.cc
+*_unittest.mm
+*_unittest.nc
+*_perftest.cc
+*[!s]_test.cc
+*[a-hj-z]s_test.cc
+*fuzz*
+*org/chromium*
+*.golden
+*.javap*
+*.pyc
+net/data/[!s]*
+net/data/s[!s]*
+net/data/ssl/[!ce]*
+net/data/ssl/c[!h]*
+net/http/transport_security_state_static.json
+net/third_party/nist-pkits
+third_party/boringssl/src/crypto/hpke/test-vectors.json
+third_party/boringssl/src/crypto/cipher_extra/test
+third_party/boringssl/src/third_party/googletest
+third_party/boringssl/src/third_party/wycheproof_testvectors

tools/import-upstream.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+set -ex
+have_version=$(cut -d= -f2 src/chrome/VERSION | tr '\n' . | cut -d. -f1-4)
+want_version=$(cat CHROMIUM_VERSION)
+if [ "$have_version" = "$want_version" ]; then
+  exit 0
+fi
+name="chromium-$want_version"
+tarball="$name.tar.xz"
+url="https://commondatastorage.googleapis.com/chromium-browser-official/$tarball"
+root=$(git rev-list --max-parents=0 HEAD)
+branch=$(git branch --show-current)
+git config core.autocrlf false
+git config core.safecrlf false
+git -c advice.detachedHead=false checkout $root
+rm -rf src
+git checkout "$branch" -- tools
+sed -i "s/^\^/$name\//" tools/include.txt
+if [ -f "/tmp/$tarball" ]; then
+  cat "/tmp/$tarball" | tar xJf - --wildcards --wildcards-match-slash -T tools/include.txt -X tools/exclude.txt
+else
+  curl "$url" -o- | tar xJf - --wildcards --wildcards-match-slash -T tools/include.txt -X tools/exclude.txt
+fi
+mv "$name" src
+git rm --quiet --force -r tools
+git add src
+git commit --quiet --amend -m "Import $name" --date=now
+git rebase --onto HEAD "$root" "$branch"

tools/include.txt

@@ -0,0 +1,73 @@
+^.clang-format
+^.gitattributes
+^.gitignore
+^.gn
+^AUTHORS
+^BUILD.gn
+^DEPS
+^LICENSE
+^base
+^build
+^build_overrides/build.gni
+^build_overrides/partition_alloc.gni
+^buildtools/deps_revisions.gni
+^buildtools/third_party/eu-strip/bin/eu-strip
+^buildtools/third_party/libc++/BUILD.gn
+^buildtools/third_party/libc++/__config_site
+^buildtools/third_party/libc++abi/BUILD.gn
+^buildtools/third_party/libc++abi/cxa_demangle_stub.cc
+^buildtools/third_party/libunwind/BUILD.gn
+^chrome/VERSION
+^chrome/android/profiles/newest.txt
+^chrome/app/theme/chromium/BRANDING
+^chrome/build/*.txt
+^chrome/version.gni
+^components/miracle_parameter
+^components/nacl/toolchain.gni
+^components/version_info
+^crypto
+^ios/features.gni
+^ipc/ipc_param_traits.h
+^net
+^testing/gtest/include/gtest/gtest_prod.h
+^third_party/abseil-cpp
+^third_party/angle/dotfile_settings.gni
+^third_party/angle/src/commit_id.py
+^third_party/angle/scripts/file_exists.py
+^third_party/apple_apsl
+^third_party/ashmem
+^third_party/boringssl
+^third_party/brotli
+^third_party/closure_compiler/closure_args.gni
+^third_party/closure_compiler/compile_js.gni
+^third_party/cpu_features
+^third_party/depot_tools/cpplint.py
+^third_party/depot_tools/download_from_google_storage.py
+^third_party/depot_tools/subprocess2.py
+^third_party/googletest/BUILD.gn
+^third_party/googletest/src/googletest/include/gtest/gtest_prod.h
+^third_party/icu/config.gni
+^third_party/jni_zero/jni_zero.gni
+^third_party/libc++
+^third_party/libc++abi
+^third_party/libevent
+^third_party/libunwind
+^third_party/lss/linux_syscall_support.h
+^third_party/modp_b64
+^third_party/nasm
+^third_party/perfetto/include/perfetto/tracing/traced_value_forward.h
+^third_party/protobuf/BUILD.gn
+^third_party/protobuf/proto_library.gni
+^third_party/protobuf/proto_sources.gni
+^third_party/protobuf/src
+^third_party/zlib
+^third_party/zstd
+^tools/cfi
+^tools/clang/scripts/update.py
+^tools/diagnosis
+^tools/grit
+^tools/gritsettings
+^tools/protoc_wrapper
+^tools/update_pgo_profiles.py
+^tools/win/DebugVisualizers
+^url

tools/list-openwrt.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+# $version can be 21.02 or 19.07.
+version=19.07.7
+if [ ! -d /tmp/openwrt ]; then
+  cd /tmp
+  git clone https://github.com/openwrt/openwrt.git
+  cd openwrt
+fi
+cd /tmp/openwrt
+git -c advice.detachedHead=false checkout v$version
+export TOPDIR=$PWD
+cd target/linux
+>targets.git
+for target in *; do
+  [ -d $target ] || continue
+  subtargets=$(make -C $target --no-print-directory DUMP=1 TARGET_BUILD=1 val.SUBTARGETS 2>/dev/null)
+  [ "$subtargets" ] || subtargets=generic
+  for subtarget in $subtargets; do
+    echo $(make -C $target --no-print-directory DUMP=1 TARGET_BUILD=1 SUBTARGET=$subtarget 2>/dev/null | egrep '^(Target:|Target-Arch-Packages:)' | cut -d: -f2) >>targets.git
+  done
+done
+
+targets=$(curl -s https://downloads.openwrt.org/releases/$version/targets/ | grep '<td class="n"><a href=' | cut -d'"' -f4 | sed 's,/,,')
+>targets.sdk
+for target in $targets; do
+  subtargets=$(curl -s https://downloads.openwrt.org/releases/$version/targets/$target/ | grep '<td class="n"><a href=' | cut -d'"' -f4 | sed 's,/,,')
+  for subtarget in $subtargets; do
+    arch=$(curl -s https://downloads.openwrt.org/releases/$version/targets/$target/$subtarget/profiles.json | grep arch_packages | cut -d'"' -f4)
+    echo $target/$subtarget $arch >>targets.sdk
+  done
+done
+
+cat >parse-targets.py <<EOF
+arch_by_target_git = {}
+arch_by_target_sdk = {}
+for line in open('targets.git'):
+    fields = line.split()
+    if not fields:
+        continue
+    arch_by_target_git[fields[0]] = fields[1]
+for line in open('targets.sdk'):
+    fields = line.split()
+    if len(fields) == 2:
+        if arch_by_target_git[fields[0]] != fields[1]:
+            raise Exception(line + ': wrong arch')
+        arch_by_target_sdk[fields[0]] = fields[1]
+    else:
+        arch_by_target_sdk[fields[0]] = ''
+for arch in sorted(set(arch_by_target_git.values())):
+    targets = []
+    for t in arch_by_target_git:
+        if arch_by_target_git[t] != arch:
+            continue
+        if t in arch_by_target_sdk:
+            targets.append(t)
+        else:
+            targets.append('~~' + t + '~~')
+    print('|', arch, '|?|', ' '.join(sorted(set(targets))), '|')
+EOF
+python3 parse-targets.py

tools/list-tls-sni-stream.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+if [ ! "$1" ]; then
+  echo "Usage: $0 PCAP_FILE"
+  exit 1
+fi
+
+file="$1"
+# Remember to disable segmentation offload so pcap files won't capture packets larger than MTU:
+# sudo ethtool --offload eth0 gso off gro off
+tshark -2 -r "$file" -R tls.handshake.extensions_server_name -T fields -e tls.handshake.extensions_server_name -e tcp.stream

tools/parse-pcap-stream.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+import os
+import sys
+import subprocess
+import yaml
+
+
+class TlsStreamParser:
+    STATE_CONTENT_TYPE = 0
+    STATE_VERSION_BYTE0 = 1
+    STATE_VERSION_BYTE1 = 2
+    STATE_LENGTH_BYTE0 = 3
+    STATE_LENGTH_BYTE1 = 4
+    STATE_DATA = 5
+
+    TLS_HEADER_SIZE = 5
+
+    def __init__(self):
+        self.state = self.STATE_CONTENT_TYPE
+        self.current_length = None
+        self.current_remaining = None
+
+    def read(self, data):
+        record_parts = []
+        i = 0
+        tls_consumed_bytes = 0
+        while i < len(data):
+            if self.state == self.STATE_CONTENT_TYPE:
+                # TODO: add content type description
+                content_type = data[i]
+                self.state = self.STATE_VERSION_BYTE0
+                i += 1
+                tls_consumed_bytes += 1
+            elif self.state == self.STATE_VERSION_BYTE0:
+                self.state = self.STATE_VERSION_BYTE1
+                i += 1
+                tls_consumed_bytes += 1
+            elif self.state == self.STATE_VERSION_BYTE1:
+                self.state = self.STATE_LENGTH_BYTE0
+                i += 1
+                tls_consumed_bytes += 1
+            elif self.state == self.STATE_LENGTH_BYTE0:
+                self.current_length = data[i]
+                self.state = self.STATE_LENGTH_BYTE1
+                i += 1
+                tls_consumed_bytes += 1
+            elif self.state == self.STATE_LENGTH_BYTE1:
+                self.current_length = self.current_length * 256 + data[i]
+                self.current_remaining = self.current_length
+                self.state = self.STATE_DATA
+                i += 1
+                tls_consumed_bytes += 1
+            elif self.state == self.STATE_DATA:
+                consume_data = min(self.current_remaining, len(data) - i)
+                self.current_remaining -= consume_data
+                i += consume_data
+                tls_consumed_bytes += consume_data
+                if self.current_remaining == 0:
+                    record_parts.append(
+                        (tls_consumed_bytes, self.TLS_HEADER_SIZE + self.current_length))
+                    tls_consumed_bytes = 0
+                    self.current_length = None
+                    self.state = self.STATE_CONTENT_TYPE
+        if tls_consumed_bytes:
+            if self.current_length is None:
+                record_parts.append((tls_consumed_bytes, '?'))
+            else:
+                record_parts.append(
+                    (tls_consumed_bytes, self.TLS_HEADER_SIZE + self.current_length))
+        return record_parts
+
+
+if len(sys.argv) != 3:
+    print(f'Usage: {sys.argv[0]} PCAP_FILE STREAM_ID')
+    os.exit(1)
+
+file = sys.argv[1]
+stream_id = sys.argv[2]
+result = subprocess.run(['tshark', '-2', '-r', file, '-q', '-z',
+                        f'follow,tcp,yaml,{stream_id}'], capture_output=True, check=True, text=True)
+
+follow_result = yaml.safe_load(result.stdout)
+LOCAL_PEER = 0
+REMOTE_PEER = 1
+assert follow_result['peers'][REMOTE_PEER][
+    'port'] == 443, f"assuming the remote peer is the TLS server: {follow_result['peers']}"
+packets = follow_result['packets']
+
+upload_stream = TlsStreamParser()
+download_stream = TlsStreamParser()
+rtt = packets[1]['timestamp'] - packets[0]['timestamp']
+time_unit = rtt / 2
+local_timestamp_first = packets[0]['timestamp']
+mitm_timestamp_first = local_timestamp_first + rtt / 4
+min_mitm_timestamp_up = packets[0]['timestamp']
+min_mitm_timestamp_down = packets[0]['timestamp']
+for packet in packets:
+    local_timestamp = packet['timestamp']
+
+    data = packet['data']
+    if packet['peer'] == LOCAL_PEER:
+        mitm_timestamp = local_timestamp + time_unit / 2
+        mitm_timestamp = max(mitm_timestamp, min_mitm_timestamp_up)
+        min_mitm_timestamp_up = mitm_timestamp
+
+        timestamp = (mitm_timestamp - mitm_timestamp_first) / time_unit
+        record_parts = upload_stream.read(data)
+        print('%.3f' % timestamp, len(data), ','.join(
+            f'{i}/{j}' for i, j in record_parts))
+    elif packet['peer'] == REMOTE_PEER:
+        mitm_timestamp = local_timestamp - time_unit / 2
+        mitm_timestamp = max(mitm_timestamp, min_mitm_timestamp_down)
+        min_mitm_timestamp_down = mitm_timestamp
+
+        timestamp = (mitm_timestamp - mitm_timestamp_first) / time_unit
+        record_parts = download_stream.read(data)
+        print('%.3f' % timestamp, -len(data),
+              ','.join(f'{i}/{j}' for i, j in record_parts))

tools/sample-traffic.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+if [ ! "$1" ]; then
+  echo "Usage: $0 IFACE"
+  exit 1
+fi
+
+iface="$1"
+sudo echo
+
+sudo tcpdump -i "$1" -s0 -w microsoft-direct.pcap &
+sleep 1
+curl https://www.example.com/
+
+sleep 1
+sudo pkill tcpdump
+
+sudo tcpdump -i "$1" -s0 -w microsoft-proxy.pcap &
+sleep 1
+curl --proxy socks5h://127.0.0.1:1080 https://www.example.com/
+
+sleep 1
+sudo pkill tcpdump &