From cace475f81c01f39096f750fabcfb09555bfbf46 Mon Sep 17 00:00:00 2001 From: klzgrad Date: Fri, 26 Apr 2019 01:40:46 +0800 Subject: [PATCH] Add continuous integration and tests --- .github/workflows/build.yml | 655 ++++++++++++++++++++++++++++++++++++ src/get-android-sys.sh | 18 + tests/basic.py | 266 +++++++++++++++ tests/basic.sh | 19 ++ 4 files changed, 958 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100755 src/get-android-sys.sh create mode 100644 tests/basic.py create mode 100755 tests/basic.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..e5c5c674fa --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,655 @@ +name: Build +on: + push: + branches: [master] + paths-ignore: [README.md] + release: + types: [published] +defaults: + run: + shell: bash + working-directory: src +env: + CACHE_EPOCH: 1 + CCACHE_MAXSIZE: 200M + CCACHE_MAXFILES: 0 + SCCACHE_CACHE_SIZE: 200M +jobs: + cache-toolchains-posix: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Cache toolchains (Linux, OpenWrt, Android) + uses: actions/cache@v2 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + src/qemu-user-static*.deb + key: toolchains-posix-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (Linux, OpenWrt) + uses: actions/cache@v2 + with: + path: src/chrome/build/pgo_profiles/ + key: pgo-linux-openwrt-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache AFDO (Android) + uses: actions/cache@v2 + with: + path: src/chrome/android/profiles/ + key: afdo-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache Android NDK (Android) + uses: actions/cache@v2 + with: + path: src/third_party/android_ndk/ + key: android-ndk-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - run: ./get-clang.sh + - run: EXTRA_FLAGS='target_os="android"' ./get-clang.sh + - run: | + if [ ! -f qemu-user-static*.deb ]; then + wget https://snapshot.debian.org/archive/debian/20220515T152741Z/pool/main/q/qemu/qemu-user-static_7.0%2Bdfsg-6_amd64.deb + fi + cache-toolchains-win: + runs-on: windows-2019 + steps: + - uses: actions/checkout@v2 + - name: Cache toolchains + uses: actions/cache@v2 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + ~/.cargo/bin/ + ~/bin/ninja.exe + key: toolchains-win-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (win64) + uses: actions/cache@v2 + with: + path: src/chrome/build/pgo_profiles/chrome-win64-* + key: pgo-win64-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (win32) + uses: actions/cache@v2 + with: + path: src/chrome/build/pgo_profiles/chrome-win32-* + key: pgo-win32-arm64-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - run: EXTRA_FLAGS='target_cpu="x64"' ./get-clang.sh + - run: EXTRA_FLAGS='target_cpu="x86"' ./get-clang.sh + - run: | + if [ ! -f ~/bin/ninja.exe ]; then + curl -LO https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip + unzip ninja-win.zip -d ~/bin + fi + cache-toolchains-mac: + runs-on: macos-11 + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/chrome/build/pgo_profiles/chrome-mac-* + src/gn/ + key: toolchains-pgo-mac-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - run: EXTRA_FLAGS='target_cpu="x64"' ./get-clang.sh + - run: EXTRA_FLAGS='target_cpu="arm64"' ./get-clang.sh + linux: + needs: cache-toolchains-posix + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + arch: [x64, x86, arm64, arm, mipsel, mips64el] + env: + EXTRA_FLAGS: 'target_cpu="${{ matrix.arch }}"' + BUNDLE: naiveproxy-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + CRONET_BUNDLE: cronet-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v3 + with: + go-version: '^1.18.1' + - name: Cache toolchains (Linux, OpenWrt, Android) + uses: actions/cache@v2 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + src/qemu-user-static*.deb + key: toolchains-posix-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (Linux, OpenWrt) + uses: actions/cache@v2 + with: + path: src/chrome/build/pgo_profiles/ + key: pgo-linux-openwrt-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache sysroot + uses: actions/cache@v2 + with: + path: src/out/sysroot-build/bullseye/bullseye_* + key: sysroot-linux-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - id: ccache-timestamp + run: echo "::set-output name=date::$(date +%s)" + - name: Cache ccache files + uses: actions/cache@v2 + with: + path: ~/.ccache + key: ccache-linux-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-${{ steps.ccache-timestamp.outputs.date }} + restore-keys: ccache-linux-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}- + - name: Install APT packages + run: | + sudo apt update + sudo apt install ninja-build pkg-config qemu-user ccache bubblewrap + sudo apt remove -y qemu-user-binfmt + sudo dpkg -i qemu-user-static_7.0+dfsg-6_amd64.deb + # libc6-i386 interferes with x86 build + sudo apt remove libc6-i386 + - run: ./get-clang.sh + - run: ccache -z + - run: ./build.sh + - run: ccache -s + - run: CCACHE_DISABLE=1 ./go-build.sh + working-directory: src/out/Release/cronet + - run: ../tests/basic.sh out/Release/naive + - name: Pack naiveproxy assets + run: | + mkdir ${{ env.BUNDLE }} + cp out/Release/naive config.json ../LICENSE ../USAGE.txt ${{ env.BUNDLE }} + tar cJf ${{ env.BUNDLE }}.tar.xz ${{ env.BUNDLE }} + openssl sha256 out/Release/naive >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUNDLE }}.tar.xz naive executable sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload naiveproxy assets + if: ${{ github.event_name == 'release' }} + run: hub release edit -a ${{ env.BUNDLE }}.tar.xz -m "" "${GITHUB_REF##*/}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Pack cronet assets + if: ${{ github.event_name == 'release' }} + run: | + mv out/Release/cronet ${{ env.CRONET_BUNDLE }} + tar cJf ${{ env.CRONET_BUNDLE }}.tar.xz ${{ env.CRONET_BUNDLE }} + openssl sha256 ${{ env.CRONET_BUNDLE }}.tar.xz >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v2 + if: ${{ github.event_name == 'release' }} + with: + name: ${{ env.CRONET_BUNDLE }}.tar.xz sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload cronet assets + if: ${{ github.event_name == 'release' }} + run: hub release edit -a ${{ env.CRONET_BUNDLE }}.tar.xz -m "" "${GITHUB_REF##*/}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + android: + needs: cache-toolchains-posix + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + arch: [x64, x86, arm64, arm] + env: + EXTRA_FLAGS: 'target_cpu="${{ matrix.arch }}" target_os="android"' + BUNDLE: naiveproxy-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + CRONET_BUNDLE: cronet-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v3 + with: + go-version: '^1.18.1' + - name: Cache toolchains (Linux, OpenWrt, Android) + uses: actions/cache@v2 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + src/qemu-user-static*.deb + key: toolchains-posix-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache AFDO (Android) + uses: actions/cache@v2 + with: + path: src/chrome/android/profiles/ + key: afdo-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache Android NDK (Android) + uses: actions/cache@v2 + with: + path: src/third_party/android_ndk/ + key: android-ndk-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache sysroot + uses: actions/cache@v2 + with: + path: src/out/sysroot-build/android/ + key: sysroot-android-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - id: ccache-timestamp + run: echo "::set-output name=date::$(date +%s)" + - name: Cache ccache files + uses: actions/cache@v2 + with: + path: ~/.ccache + key: ccache-android-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-${{ steps.ccache-timestamp.outputs.date }} + restore-keys: ccache-android-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}- + - name: Install APT packages + run: | + sudo apt update + sudo apt install ninja-build pkg-config qemu-user ccache bubblewrap + sudo apt remove -y qemu-user-binfmt + sudo dpkg -i qemu-user-static_7.0+dfsg-6_amd64.deb + # libc6-i386 interferes with x86 build + sudo apt remove libc6-i386 + - run: ./get-clang.sh + - run: ccache -z + - run: ./build.sh + - run: ccache -s + - run: ./get-android-sys.sh + - run: CCACHE_DISABLE=1 ./go-build.sh + working-directory: src/out/Release/cronet + - run: ../tests/basic.sh out/Release/naive + - name: Pack naiveproxy assets + run: | + mkdir ${{ env.BUNDLE }} + cp out/Release/naive config.json ../LICENSE ../USAGE.txt ${{ env.BUNDLE }} + tar cJf ${{ env.BUNDLE }}.tar.xz ${{ env.BUNDLE }} + openssl sha256 out/Release/naive >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUNDLE }}.tar.xz naive executable sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload naiveproxy assets + if: ${{ github.event_name == 'release' }} + run: hub release edit -a ${{ env.BUNDLE }}.tar.xz -m "" "${GITHUB_REF##*/}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Pack cronet assets + if: ${{ github.event_name == 'release' }} + run: | + mv out/Release/cronet ${{ env.CRONET_BUNDLE }} + tar cJf ${{ env.CRONET_BUNDLE }}.tar.xz ${{ env.CRONET_BUNDLE }} + openssl sha256 ${{ env.CRONET_BUNDLE }}.tar.xz >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v2 + if: ${{ github.event_name == 'release' }} + with: + name: ${{ env.CRONET_BUNDLE }}.tar.xz sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload cronet assets + if: ${{ github.event_name == 'release' }} + run: hub release edit -a ${{ env.CRONET_BUNDLE }}.tar.xz -m "" "${GITHUB_REF##*/}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + win: + needs: cache-toolchains-win + runs-on: windows-2019 + strategy: + fail-fast: false + matrix: + arch: [x64, x86, arm64] + env: + EXTRA_FLAGS: 'target_cpu="${{ matrix.arch }}"' + BUNDLE: naiveproxy-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + CRONET_BUNDLE: cronet-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v3 + with: + go-version: '^1.18.1' + - name: Cache toolchains + uses: actions/cache@v2 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + ~/.cargo/bin/ + ~/bin/ninja.exe + key: toolchains-win-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (win64) + if: ${{ matrix.arch == 'x64' }} + uses: actions/cache@v2 + with: + path: src/chrome/build/pgo_profiles/chrome-win64-* + key: pgo-win64-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (win32) + if: ${{ matrix.arch != 'x64' }} + uses: actions/cache@v2 + with: + path: src/chrome/build/pgo_profiles/chrome-win32-* + key: pgo-win32-arm64-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - id: ccache-timestamp + run: echo "::set-output name=date::$(date +%s)" + - name: Cache ccache files + uses: actions/cache@v2 + with: + path: ~/AppData/Local/Mozilla/sccache + key: ccache-win-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-${{ steps.ccache-timestamp.outputs.date }} + restore-keys: ccache-win-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}- + - run: ./get-clang.sh + - run: ~/.cargo/bin/sccache -z + - run: ./build.sh + - run: ~/.cargo/bin/sccache -s + - run: CCACHE_DISABLE=1 ./go-build.sh + working-directory: src/out/Release/cronet + - run: ../tests/basic.sh out/Release/naive + # No real or emulated environment is available to test this. + if: ${{ matrix.arch != 'arm64' }} + - name: Pack naiveproxy assets + run: | + mkdir ${{ env.BUNDLE }} + cp out/Release/naive config.json ../LICENSE ../USAGE.txt ${{ env.BUNDLE }} + 7z a ${{ env.BUNDLE }}.zip ${{ env.BUNDLE }} + openssl sha256 out/Release/naive.exe >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUNDLE }}.zip naive executable sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload naiveproxy assets + if: ${{ github.event_name == 'release' }} + run: hub release edit -a ${{ env.BUNDLE }}.zip -m "" "${GITHUB_REF##*/}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Pack cronet assets + if: ${{ github.event_name == 'release' }} + run: | + mv out/Release/cronet ${{ env.CRONET_BUNDLE }} + 7z a ${{ env.CRONET_BUNDLE }}.zip ${{ env.CRONET_BUNDLE }} + openssl sha256 ${{ env.CRONET_BUNDLE }}.zip >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v2 + if: ${{ github.event_name == 'release' }} + with: + name: ${{ env.CRONET_BUNDLE }}.zip sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload cronet assets + if: ${{ github.event_name == 'release' }} + run: hub release edit -a ${{ env.CRONET_BUNDLE }}.zip -m "" "${GITHUB_REF##*/}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + mac: + needs: cache-toolchains-mac + runs-on: macos-11 + strategy: + fail-fast: false + matrix: + arch: [x64, arm64] + env: + EXTRA_FLAGS: 'target_cpu="${{ matrix.arch }}"' + BUNDLE: naiveproxy-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + CRONET_BUNDLE: cronet-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v3 + with: + go-version: '^1.18.1' + - name: Cache toolchains and PGO + uses: actions/cache@v2 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/chrome/build/pgo_profiles/chrome-mac-* + src/gn/ + key: toolchains-pgo-mac-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - id: ccache-timestamp + run: echo "::set-output name=date::$(date +%s)" + - name: Cache ccache files + uses: actions/cache@v2 + with: + path: ~/Library/Caches/ccache + key: ccache-mac-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-${{ steps.ccache-timestamp.outputs.date }} + restore-keys: ccache-mac-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}- + - run: brew install ninja ccache + - run: ./get-clang.sh + - run: ccache -z + - run: ./build.sh + - run: ccache -s + - run: CCACHE_DISABLE=1 ./go-build.sh + working-directory: src/out/Release/cronet + - run: ../tests/basic.sh out/Release/naive + # No real or emulated environment is available to test this. + if: ${{ matrix.arch != 'arm64' }} + - name: Pack naiveproxy assets + run: | + mkdir ${{ env.BUNDLE }} + cp out/Release/naive config.json ../LICENSE ../USAGE.txt ${{ env.BUNDLE }} + tar cJf ${{ env.BUNDLE }}.tar.xz ${{ env.BUNDLE }} + openssl sha256 out/Release/naive >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUNDLE }}.tar.xz naive executable sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload naiveproxy assets + if: ${{ github.event_name == 'release' }} + run: hub release edit -a ${{ env.BUNDLE }}.tar.xz -m "" "${GITHUB_REF##*/}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Pack cronet assets + if: ${{ github.event_name == 'release' }} + run: | + mv out/Release/cronet ${{ env.CRONET_BUNDLE }} + tar cJf ${{ env.CRONET_BUNDLE }}.tar.xz ${{ env.CRONET_BUNDLE }} + openssl sha256 ${{ env.CRONET_BUNDLE }}.tar.xz >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v2 + if: ${{ github.event_name == 'release' }} + with: + name: ${{ env.CRONET_BUNDLE }}.tar.xz sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload cronet assets + if: ${{ github.event_name == 'release' }} + run: hub release edit -a ${{ env.CRONET_BUNDLE }}.tar.xz -m "" "${GITHUB_REF##*/}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ios: + needs: cache-toolchains-mac + runs-on: macos-11 + strategy: + fail-fast: false + matrix: + arch: [x64, arm64] + env: + EXTRA_FLAGS: 'target_cpu="${{ matrix.arch }}" target_os="ios" ios_enable_code_signing=false' + BUNDLE: naiveproxy-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + steps: + - uses: actions/checkout@v2 + - name: Cache toolchains and PGO + uses: actions/cache@v2 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/chrome/build/pgo_profiles/chrome-mac-* + src/gn/ + key: toolchains-pgo-mac-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - id: ccache-timestamp + run: echo "::set-output name=date::$(date +%s)" + - name: Cache ccache files + uses: actions/cache@v2 + with: + path: ~/Library/Caches/ccache + key: ccache-ios-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-${{ steps.ccache-timestamp.outputs.date }} + restore-keys: ccache-ios-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}- + - run: brew install ninja ccache + - run: ./get-clang.sh + - run: ccache -z + - run: ./build.sh + - run: ccache -s + openwrt: + needs: cache-toolchains-posix + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + openwrt: 'target=x86 subtarget=64' + target_cpu: x64 + - arch: x86 + openwrt: 'target=x86 subtarget=generic' + target_cpu: x86 + - arch: aarch64_cortex-a53 + openwrt: 'target=sunxi subtarget=cortexa53' + target_cpu: arm64 + extra: 'arm_cpu="cortex-a53"' + - arch: aarch64_cortex-a53-static + openwrt: 'target=sunxi subtarget=cortexa53' + target_cpu: arm64 + extra: 'arm_cpu="cortex-a53" build_static=true' + - arch: aarch64_cortex-a72 + openwrt: 'target=mvebu subtarget=cortexa72' + target_cpu: arm64 + extra: 'arm_cpu="cortex-a72"' + - arch: aarch64_generic + openwrt: 'target=rockchip subtarget=armv8' + target_cpu: arm64 + - arch: arm_arm1176jzf-s_vfp + openwrt: 'target=bcm27xx subtarget=bcm2708' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="arm1176jzf-s" arm_fpu="vfp" arm_float_abi="hard" arm_use_neon=false arm_use_thumb=false' + - arch: arm_arm926ej-s + openwrt: 'target=mxs' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="arm926ej-s" arm_float_abi="soft" arm_use_neon=false arm_use_thumb=false' + - arch: arm_cortex-a15_neon-vfpv4 + openwrt: 'target=armvirt subtarget=32' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a15" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true' + - arch: arm_cortex-a5_vfpv4 + openwrt: 'target=at91 subtarget=sama5' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a5" arm_fpu="vfpv4" arm_float_abi="hard" arm_use_neon=false' + - arch: arm_cortex-a7 + openwrt: 'target=mediatek subtarget=mt7629' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a7" arm_float_abi="soft" arm_use_neon=false' + - arch: arm_cortex-a7_neon-vfpv4 + openwrt: 'target=sunxi subtarget=cortexa7' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a7" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true' + - arch: arm_cortex-a7_neon-vfpv4-static + openwrt: 'target=sunxi subtarget=cortexa7' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a7" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true build_static=true' + - arch: arm_cortex-a8_vfpv3 + openwrt: 'target=sunxi subtarget=cortexa8' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="vfpv3" arm_float_abi="hard" arm_use_neon=false' + - arch: arm_cortex-a9 + openwrt: 'target=bcm53xx subtarget=generic' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a9" arm_float_abi="soft" arm_use_neon=false' + - arch: arm_cortex-a9-static + openwrt: 'target=bcm53xx subtarget=generic' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a9" arm_float_abi="soft" arm_use_neon=false build_static=true' + - arch: arm_cortex-a9_neon + openwrt: 'target=imx6' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true' + - arch: arm_cortex-a9_vfpv3-d16 + openwrt: 'target=tegra' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="vfpv3-d16" arm_float_abi="hard" arm_use_neon=false' + - arch: arm_mpcore + openwrt: 'target=oxnas subtarget=ox820' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="mpcore" arm_float_abi="soft" arm_use_neon=false arm_use_thumb=false' + - arch: arm_xscale + openwrt: 'target=kirkwood' + target_cpu: arm + extra: 'arm_version=0 arm_cpu="xscale" arm_float_abi="soft" arm_use_neon=false arm_use_thumb=false' + - arch: mipsel_24kc + openwrt: 'target=ramips subtarget=rt305x' + target_cpu: mipsel + extra: 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="24kc"' + - arch: mipsel_74kc + openwrt: 'target=ramips subtarget=rt3883' + target_cpu: mipsel + extra: 'mips_arch_variant="r2" mips_float_abi="soft" mips_tune="74kc"' + - arch: mipsel_mips32 + openwrt: 'target=bcm47xx subtarget=generic' + target_cpu: mipsel + extra: 'mips_arch_variant="r1" mips_float_abi="soft"' + env: + EXTRA_FLAGS: target_cpu="${{ matrix.target_cpu }}" target_os="openwrt" use_allocator="none" use_allocator_shim=false use_partition_alloc=false ${{ matrix.extra }} + OPENWRT_FLAGS: arch=${{ matrix.arch }} release=21.02.2 gcc_ver=8.4.0 ${{ matrix.openwrt }} + BUNDLE: naiveproxy-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + CRONET_BUNDLE: cronet-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v3 + with: + go-version: '^1.18.1' + - name: Cache toolchains (Linux, OpenWrt, Android) + uses: actions/cache@v2 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + src/qemu-user-static*.deb + key: toolchains-posix-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (Linux, OpenWrt) + uses: actions/cache@v2 + with: + path: src/chrome/build/pgo_profiles/ + key: pgo-linux-openwrt-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache sysroot + uses: actions/cache@v2 + with: + path: src/out/sysroot-build/openwrt + key: sysroot-openwrt-21.02.2-${{ matrix.arch }}-v${{ env.CACHE_EPOCH }} + - id: ccache-timestamp + run: echo "::set-output name=date::$(date +%s)" + - name: Cache ccache files + uses: actions/cache@v2 + with: + path: ~/.ccache + key: ccache-openwrt-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-${{ steps.ccache-timestamp.outputs.date }} + restore-keys: ccache-openwrt-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}- + - name: Install APT packages + run: | + sudo apt update + sudo apt install ninja-build pkg-config qemu-user ccache bubblewrap + sudo apt remove -y qemu-user-binfmt + sudo dpkg -i qemu-user-static_7.0+dfsg-6_amd64.deb + # libc6-i386 interferes with x86 build + sudo apt remove libc6-i386 + - run: ./get-clang.sh + - run: ccache -z + - run: ./build.sh + - run: ccache -s + - run: CCACHE_DISABLE=1 ./go-build.sh + working-directory: src/out/Release/cronet + if: ${{ ! contains(matrix.extra, 'build_static=true') }} + - run: ../tests/basic.sh out/Release/naive + - name: Pack naiveproxy assets + run: | + mkdir ${{ env.BUNDLE }} + cp out/Release/naive config.json ../LICENSE ../USAGE.txt ${{ env.BUNDLE }} + tar cJf ${{ env.BUNDLE }}.tar.xz ${{ env.BUNDLE }} + openssl sha256 out/Release/naive >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUNDLE }}.tar.xz naive executable sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload naiveproxy assets + if: ${{ github.event_name == 'release' }} + run: hub release edit -a ${{ env.BUNDLE }}.tar.xz -m "" "${GITHUB_REF##*/}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Pack cronet assets + if: ${{ github.event_name == 'release' && ! contains(matrix.extra, 'build_static=true') }} + run: | + mv out/Release/cronet ${{ env.CRONET_BUNDLE }} + tar cJf ${{ env.CRONET_BUNDLE }}.tar.xz ${{ env.CRONET_BUNDLE }} + openssl sha256 ${{ env.CRONET_BUNDLE }}.tar.xz >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v2 + if: ${{ github.event_name == 'release' && ! contains(matrix.extra, 'build_static=true') }} + with: + name: ${{ env.CRONET_BUNDLE }}.tar.xz sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload cronet assets + if: ${{ github.event_name == 'release' && ! contains(matrix.extra, 'build_static=true') }} + run: hub release edit -a ${{ env.CRONET_BUNDLE }}.tar.xz -m "" "${GITHUB_REF##*/}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/src/get-android-sys.sh b/src/get-android-sys.sh new file mode 100755 index 0000000000..3913b70229 --- /dev/null +++ b/src/get-android-sys.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -ex + +. ./get-sysroot.sh + +if [ "$WITH_ANDROID_IMG" -a ! -d out/sysroot-build/android/"$WITH_ANDROID_IMG"/system ]; then + curl -O https://dl.google.com/android/repository/sys-img/android/$WITH_ANDROID_IMG.zip + mkdir -p $WITH_ANDROID_IMG/mount + unzip $WITH_ANDROID_IMG.zip '*/system.img' -d $WITH_ANDROID_IMG + sudo mount $WITH_ANDROID_IMG/*/system.img $WITH_ANDROID_IMG/mount + rootfs=out/sysroot-build/android/$WITH_ANDROID_IMG + mkdir -p $rootfs/system/bin $rootfs/system/etc + cp $WITH_ANDROID_IMG/mount/bin/linker* $rootfs/system/bin + cp $WITH_ANDROID_IMG/mount/etc/hosts $rootfs/system/etc + cp -r $WITH_ANDROID_IMG/mount/lib* $rootfs/system + sudo umount $WITH_ANDROID_IMG/mount + rm -rf $WITH_ANDROID_IMG $WITH_ANDROID_IMG.zip +fi diff --git a/tests/basic.py b/tests/basic.py new file mode 100644 index 0000000000..5584e2511d --- /dev/null +++ b/tests/basic.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +import argparse +import http.server +import os +import shutil +import ssl +import subprocess +import tempfile +import threading +import time + +parser = argparse.ArgumentParser() +parser.add_argument('--naive', required=True) +parser.add_argument('--rootfs') +parser.add_argument('--target_cpu') +argv = parser.parse_args() + +if argv.rootfs: + try: + os.remove(os.path.join(argv.rootfs, 'naive')) + except OSError: + pass + +_, certfile = tempfile.mkstemp() + +result = subprocess.run( + f'openssl req -new -x509 -keyout {certfile} -out {certfile} -days 1 -nodes -subj /C=XX'.split(), capture_output=True) +result.check_returncode() + +HTTPS_SERVER_HOSTNAME = '127.0.0.1' +HTTP_SERVER_PORT = 60443 +ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +ssl_context.load_cert_chain(certfile=certfile) +httpd = http.server.HTTPServer( + (HTTPS_SERVER_HOSTNAME, HTTP_SERVER_PORT), http.server.SimpleHTTPRequestHandler) +httpd.timeout = 1 +httpd.allow_reuse_address = True +httpd.socket = ssl_context.wrap_socket(httpd.socket, server_side=True) + +httpd_thread = threading.Thread( + target=lambda httpd: httpd.serve_forever(), args=(httpd,), daemon=True) +httpd_thread.start() + + +def test_https_server(hostname, port, proxy=None): + url = f'https://{hostname}:{port}/404' + cmdline = ['curl', '-k', '-s'] + if proxy: + cmdline.extend(['--proxy', proxy]) + cmdline.append(url) + print('subprocess.run', ' '.join(cmdline)) + result = subprocess.run(cmdline, capture_output=True, + timeout=1, text=True, encoding='utf-8') + print(result.stderr, end='') + return 'Error code: 404' in result.stdout + + +assert test_https_server(HTTPS_SERVER_HOSTNAME, + HTTP_SERVER_PORT), 'https server not up' + + +def start_naive(naive_args): + with_qemu = None + if argv.target_cpu == 'arm64': + with_qemu = 'aarch64' + elif argv.target_cpu == 'arm': + with_qemu = 'arm' + elif argv.target_cpu == 'mipsel': + with_qemu = 'mipsel' + elif argv.target_cpu == 'mips64el': + with_qemu = 'mips64el' + + if argv.rootfs: + if not with_qemu: + if not os.path.exists(os.path.join(argv.rootfs, 'naive')): + shutil.copy2(argv.naive, argv.rootfs) + cmdline = ['bwrap', '--die-with-parent', '--bind', argv.rootfs, '/', + '--proc', '/proc', '--dev', '/dev', '/naive'] + else: + cmdline = [f'qemu-{with_qemu}', '-L', argv.rootfs, argv.naive] + else: + cmdline = [argv.naive] + cmdline.extend(naive_args) + + proc = subprocess.Popen(cmdline, stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, text=True, encoding='utf-8') + print('subprocess.Popen', ' '.join(cmdline), 'pid:', proc.pid) + + def terminate(proc): + print('proc has timed out') + print('terminate pid', proc.pid) + proc.terminate() + + timeout = threading.Timer(10, terminate, args=(proc,)) + timeout.start() + while True: + if proc.poll() is not None: + timeout.cancel() + return proc.poll() == 0 + + line = proc.stderr.readline().strip() + print(line) + if 'Failed to listen: ' in line: + timeout.cancel() + print('terminate pid', proc.pid) + proc.terminate() + return 'Failed to listen' + elif 'Listening on ' in line: + timeout.cancel() + return proc + + +port = 10000 + + +def allocate_port_number(): + global port + port += 1 + if port > 60000: + port = 10000 + return port + + +def test_naive_once(proxy, *args, **kwargs): + port_map = {} + + class PortDict(dict): + def __init__(self, port_map): + self._port_map = port_map + + def __getitem__(self, key): + if key.startswith('PORT'): + if key not in self._port_map: + self._port_map[key] = str(allocate_port_number()) + return self._port_map[key] + return key + port_dict = PortDict(port_map) + + proxy = proxy.format_map(port_dict) + + config_file = kwargs.get('config_file', 'config.json') + if argv.rootfs: + config_file = os.path.join(argv.rootfs, config_file) + config_content = kwargs.get('config_content') + if config_content is not None: + config_content = config_content.format_map(port_dict) + with open(config_file, 'w') as f: + f.write('{') + f.write(config_content) + f.write('}') + + naive_procs = [] + + def cleanup(): + if config_content is not None: + os.remove(config_file) + for naive_proc in naive_procs: + print('terminate pid', naive_proc.pid) + naive_proc.terminate() + + for args_instance in args: + naive_args = args_instance.format_map(port_dict).split() + naive_proc = start_naive(naive_args) + if naive_proc == 'Failed to listen': + cleanup() + return 'Failed to listen' + if not naive_proc: + cleanup() + return False + naive_procs.append(naive_proc) + + result = test_https_server(HTTPS_SERVER_HOSTNAME, HTTP_SERVER_PORT, proxy) + + cleanup() + + return result + + +def test_naive(label, proxy, *args, **kwargs): + RETRIES = 5 + for i in range(RETRIES): + result = test_naive_once(proxy, *args, **kwargs) + if result == 'Failed to listen': + print('Retrying...') + time.sleep(1) + continue + if result: + print('** TEST PASS:', label, end='\n\n') + return True + return result + print('** TEST FAIL:', label, end='\n\n') + os.exit(1) + + +test_naive('Default config', 'socks5h://127.0.0.1:1080', + '--log') + +test_naive('Default config file', 'socks5h://127.0.0.1:{PORT1}', + '', + config_content='"listen":"socks://127.0.0.1:{PORT1}","log":""') + +test_naive('Custom config file', 'socks5h://127.0.0.1:{PORT1}', + 'custom.json', + config_content='"listen":"socks://127.0.0.1:{PORT1}","log":""', + config_file='custom.json') + +test_naive('Trivial - listen scheme only', 'socks5h://127.0.0.1:1080', + '--log --listen=socks://') + +test_naive('Trivial - listen no host', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1}') + +test_naive('Trivial - listen no port', 'socks5h://127.0.0.1:1080', + '--log --listen=socks://127.0.0.1') + +test_naive('Trivial - auth', 'socks5h://user:pass@127.0.0.1:{PORT1}', + '--log --listen=socks://user:pass@127.0.0.1:{PORT1}') + +test_naive('Trivial - auth with special chars', 'socks5h://user:^@127.0.0.1:{PORT1}', + '--log --listen=socks://user:^@127.0.0.1:{PORT1}') + +test_naive('Trivial - auth with special chars', 'socks5h://^:^@127.0.0.1:{PORT1}', + '--log --listen=socks://^:^@127.0.0.1:{PORT1}') + +test_naive('Trivial - auth with empty pass', 'socks5h://user:@127.0.0.1:{PORT1}', + '--log --listen=socks://user:@127.0.0.1:{PORT1}') + +test_naive('SOCKS-SOCKS', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1} --proxy=socks://127.0.0.1:{PORT2}', + '--log --listen=socks://:{PORT2}') + +test_naive('SOCKS-SOCKS - proxy no port', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1} --proxy=socks://127.0.0.1', + '--log --listen=socks://:1080') + +test_naive('SOCKS-HTTP', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1} --proxy=http://127.0.0.1:{PORT2}', + '--log --listen=http://:{PORT2}') + +test_naive('HTTP-HTTP', 'http://127.0.0.1:{PORT1}', + '--log --listen=http://:{PORT1} --proxy=http://127.0.0.1:{PORT2}', + '--log --listen=http://:{PORT2}') + +test_naive('HTTP-SOCKS', 'http://127.0.0.1:{PORT1}', + '--log --listen=http://:{PORT1} --proxy=socks://127.0.0.1:{PORT2}', + '--log --listen=socks://:{PORT2}') + +test_naive('SOCKS-SOCKS-SOCKS', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1} --proxy=socks://127.0.0.1:{PORT2}', + '--log --listen=socks://:{PORT2} --proxy=socks://127.0.0.1:{PORT3}', + '--log --listen=socks://:{PORT3}') + +test_naive('SOCKS-HTTP-SOCKS', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1} --proxy=http://127.0.0.1:{PORT2}', + '--log --listen=http://:{PORT2} --proxy=socks://127.0.0.1:{PORT3}', + '--log --listen=socks://:{PORT3}') + +test_naive('HTTP-SOCKS-HTTP', 'http://127.0.0.1:{PORT1}', + '--log --listen=http://:{PORT1} --proxy=socks://127.0.0.1:{PORT2}', + '--log --listen=socks://:{PORT2} --proxy=http://127.0.0.1:{PORT3}', + '--log --listen=http://:{PORT3}') + +test_naive('HTTP-HTTP-HTTP', 'http://127.0.0.1:{PORT1}', + '--log --listen=http://:{PORT1} --proxy=http://127.0.0.1:{PORT2}', + '--log --listen=http://:{PORT2} --proxy=http://127.0.0.1:{PORT3}', + '--log --listen=http://:{PORT3}') diff --git a/tests/basic.sh b/tests/basic.sh new file mode 100755 index 0000000000..76dd15d900 --- /dev/null +++ b/tests/basic.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -ex + +script_dir=$(dirname "$PWD/$0") + +[ "$1" ] || exit 1 +naive="$PWD/$1" + +. ./get-sysroot.sh + +if [ "$WITH_ANDROID_IMG" ]; then + rootfs="$PWD/out/sysroot-build/android/$WITH_ANDROID_IMG" +elif [ "$WITH_SYSROOT" ]; then + rootfs="$PWD/$WITH_SYSROOT" +fi + +cd /tmp +python3 "$script_dir"/basic.py --naive="$naive" --rootfs="$rootfs" --target_cpu="$target_cpu"